{
  "annotation_id": "a688f047-8976-4ad2-ba37-c003c593ee81",
  "date_created": "2026-02-17T05:53:07.762000Z",
  "date_modified": "2026-02-17T05:53:07.762000Z",
  "file_hash": "e40eb5ab9b04fa0bcb026d0ab7ae152fc183922932cacb6ad9037e74e47fd19f",
  "private": false,
  "record": {
    "abstract": "Large language models are increasingly applied to various development scenarios. However, in on-chain transaction scenarios, even a minor error can cause irreversible loss for users. Existing evaluations often overlook execution accuracy and safety. We introduce EVM-QuestBench, an execution-grounded benchmark for natural-language transaction-script generation on EVM-compatible chains. The benchmark employs dynamic evaluation: instructions are sampled from template pools, numeric parameters are drawn from predefined intervals, and validators verify outcomes against these instantiated values. EVM-QuestBench contains 107 tasks (62 atomic, 45 composite). Its modular architecture enables rapid task development. The runner executes scripts on a forked EVM chain with snapshot isolation; composite tasks apply step-efficiency decay. We evaluate 20 models and find large performance gaps, with split scores revealing persistent asymmetry between single-action precision and multi-step workflow completion. Code: https://anonymous.4open.science/r/bsc_quest_bench-A9CF/.",
    "arxiv_id": "2601.06565",
    "authors": [
      "Pei Yang",
      "Wanyi Chen",
      "Ke Wang",
      "Lynn Ai",
      "Eric Yang",
      "Tianyu Shi"
    ],
    "categories": [
      "cs.CL"
    ],
    "license": "http://creativecommons.org/licenses/by/4.0/",
    "title": "EVM-QuestBench: An Execution-Grounded Benchmark for Natural-Language Transaction Code Generation",
    "url": "https://arxiv.org/abs/2601.06565",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "265b2c57-2dbd-4ab8-a72b-37b367b7ecc1",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}