{
  "annotation_id": "df06a9ae-eef0-4d49-89db-109187f84a7d",
  "date_created": "2026-02-17T05:53:24.342000Z",
  "date_modified": "2026-02-17T05:53:24.342000Z",
  "file_hash": "316be70cfb0416b2fef123192ebaca09b0eb9afe1746244b8a8a08cf28b0462e",
  "private": false,
  "record": {
    "abstract": "The practice of fine-tuning Large Language Models (LLMs) has achieved state-of-the-art performance on specialized tasks, yet diagnosing why these models become brittle and fail to generalize remains a critical open problem. To address this, we introduce and apply a multi-layered diagnostic framework to a cross-architectural study. We fine-tune Llama 3.1 8B, Gemma 2 9B, and Mistral models on a high-stakes phishing detection task and use SHAP analysis and mechanistic interpretability to uncover the root causes of their generalization failures. Our investigation reveals three critical findings: (1) Generalization is driven by a powerful synergy between architecture and data diversity. The Gemma 2 9B model achieves state-of-the-art performance (\u003e91\\% F1), but only when trained on a stylistically diverse ``generalist\u0027\u0027 dataset. (2) Generalization is highly architecture-dependent. We diagnose a specific failure mode in Llama 3.1 8B, which performs well on a narrow domain but cannot integrate diverse data, leading to a significant performance drop. (3) Some architectures are inherently more generalizable. The Mistral model proves to be a consistent and resilient performer across multiple training paradigms. By pinpointing the flawed heuristics responsible for these failures, our work provides a concrete methodology for diagnosing and understanding generalization failures, underscoring that reliable AI requires deep validation of the interplay between architecture, data, and training strategy.",
    "arxiv_id": "2601.10524",
    "authors": [
      "Frank Bobe III",
      "Gregory D. Vetaw",
      "Chase Pavlick",
      "Darshan Bryner",
      "Matthew Cook",
      "Jose Salas-Vernis"
    ],
    "categories": [
      "cs.AI"
    ],
    "license": "http://creativecommons.org/publicdomain/zero/1.0/",
    "title": "Diagnosing Generalization Failures in Fine-Tuned LLMs: A Cross-Architectural Study on Phishing Detection",
    "url": "https://arxiv.org/abs/2601.10524",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "2d80fe97-94c0-45cb-927a-b06897d30941",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}