{
  "annotation_id": "b90b69d1-249f-4e59-8aee-34c5fdfc4ea3",
  "date_created": "2026-02-17T05:53:20.022000Z",
  "date_modified": "2026-02-17T05:53:20.022000Z",
  "file_hash": "1cd8a0bfae56836b69f1154b03f022ee3c93e8e190cdaf04f12a0ddebb42f925",
  "private": false,
  "record": {
    "abstract": "Flaky tests are a common problem in software testing. They produce inconsistent results when executed multiple times on the same code, invalidating the assumption that a test failure indicates a software defect. Recent work on LLM-based test generation has identified flakiness as a potential problem with generated tests. However, its prevalence and underlying causes are unclear. We examined the flakiness of LLM-generated tests in the context of four relational database management systems: SAP HANA, DuckDB, MySQL, and SQLite. We amplified test suites with two LLMs, GPT-4o and Mistral-Large-Instruct-2407, to assess the flakiness of the generated test cases. Our results suggest that generated tests have a slightly higher proportion of flaky tests compared to existing tests. Based on a manual inspection, we found that the most common root cause of flakiness was the reliance of a test on a certain order that is not guaranteed (\"unordered collection\"), which was present in 72 of 115 flaky tests (63%). Furthermore, both LLMs transferred the flakiness from the existing tests to the newly generated tests via the provided prompt context. Our experiments suggest that flakiness transfer is more prevalent in closed-source systems such as SAP HANA than in open-source systems. Our study informs developers on what types of flakiness to expect from LLM-generated tests. It also highlights the importance of providing LLMs with tailored context when employing LLMs for test generation.",
    "arxiv_id": "2601.08998",
    "authors": [
      "Alexander Berndt",
      "Thomas Bach",
      "Rainer Gemulla",
      "Marcus Kessel",
      "Sebastian Baltes"
    ],
    "categories": [
      "cs.SE"
    ],
    "license": "http://creativecommons.org/licenses/by/4.0/",
    "title": "On the Flakiness of LLM-Generated Tests for Industrial and Open-Source Database Management Systems",
    "url": "https://arxiv.org/abs/2601.08998",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "24cf0380-79d8-4177-ac11-b8fcbf52f615",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}