{
  "annotation_id": "9385c33f-54aa-45d8-906b-9a69fce04b9c",
  "date_created": "2026-02-17T05:53:20.238000Z",
  "date_modified": "2026-02-17T05:53:20.238000Z",
  "file_hash": "ca57b0dea729f113fd753ae217e01f1737fc6b4114dcab0329927510f20aaf32",
  "private": false,
  "record": {
    "abstract": "Recent advancements in large language models (LLMs) have significantly enhanced their reasoning capabilities. However, they continue to struggle with basic character-level tasks, such as counting letters in words, a problem rooted in their tokenization process. While existing benchmarks have highlighted this weakness through basic character operations, such failures are often dismissed due to lacking practical relevance. Yet, many real-world applications, such as navigating text-based maps or interpreting structured tables, rely heavily on precise sub-token understanding. In this regard, we introduce SubTokenTest, a comprehensive benchmark that assesses sub-token understanding through practical, utility-driven tasks. Our benchmark includes ten tasks across four domains and isolates tokenization-related failures by decoupling performance from complex reasoning. We provide a comprehensive evaluation of nine advanced LLMs. Additionally, we investigate the impact of test-time scaling on sub-token reasoning and explore how character-level information is encoded within the hidden states.",
    "arxiv_id": "2601.09089",
    "authors": [
      "Shuyang Hou",
      "Yi Hu",
      "Muhan Zhang"
    ],
    "categories": [
      "cs.CL",
      "cs.AI"
    ],
    "license": "http://arxiv.org/licenses/nonexclusive-distrib/1.0/",
    "title": "SubTokenTest: A Practical Benchmark for Real-World Sub-token Understanding",
    "url": "https://arxiv.org/abs/2601.09089",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "a0e42b2e-d46c-4bae-aa82-5303b4e36c47",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}