{
  "review_date_utc": "2026-10-25",
  "research_dates": {
    "A": "2026-10-22",
    "B": "2026-10-22",
    "C": "2026-10-25"
  },
  "scope": "Full reading of 33 drafts; targeted live verification, not exhaustive fact verification. At least three substantive claims per article plus every discovered material changing specification/pricing and numerical research-result claim. Bibliography associations audited separately from substantive claim counts.",
  "method": "Original papers, maintainer pages and official documentation. Arithmetic independently recalculated; no model-performance experiment reproduced. Editorial recommendations assessed for support and boundaries, not falsely treated as experimental findings.",
  "verdict_definitions": {
    "supported": "Source passage supports claim within stated scope.",
    "wrong_bibliographic_metadata": "Real relevant source, inaccurate attribution label; not fabricated evidence.",
    "partially_supported": "Some substance supported; cited source or scope does not fully match.",
    "unsupported_guidance": "Advice exceeds cited evidence; not automatically false or fabricated.",
    "incorrect_guidance": "Concrete reasoning or method error with reader consequences.",
    "overstated_evidence": "Evidence supports a qualified statement, not the absolute claim.",
    "inaccurate_version_label": "Real result associated with the wrong snapshot/first-release label.",
    "missing_material_condition": "Correct quoted value with an omitted condition that changes applicability.",
    "needs_clarification": "Ambiguity or overgeneralisation needing a bounded statement.",
    "supported_with_citation_gap": "Correct checked method but article lacks a relevant nearby reference.",
    "inconsistent_guidance": "Conflicting operational rules within the cluster."
  },
  "source_catalog": [
    {
      "source_id": "eval",
      "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "title": "Evaluation best practices",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice",
      "observation": "Task-specific datasets, human calibration, continuous evaluation and judge controls; Evals read-only 31 October and shutdown 30 November 2026.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "graders",
      "url": "https://developers.openai.com/api/docs/guides/graders",
      "title": "Graders",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Grader types; Model graders; Grader hacking",
      "observation": "String, similarity, model and Python graders; compare expert quality ordering; audit reward hacking.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "agentdocs",
      "url": "https://developers.openai.com/api/docs/guides/agent-evals",
      "title": "Evaluate agent workflows",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Start with traces; Move to datasets and eval runs",
      "observation": "Traces expose calls, guardrails and handoffs; structured grading and repeatable datasets have different roles.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "reason",
      "url": "https://developers.openai.com/api/docs/guides/reasoning",
      "title": "Reasoning models",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning",
      "observation": "Effort is adaptive and model-dependent. Reasoning is billed within output usage; counts are exposed. Initial experimental buffer recommendation: 25,000 tokens.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "price",
      "url": "https://developers.openai.com/api/docs/pricing",
      "title": "Pricing",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Flagship Standard and Flex/Batch tables; Tools",
      "observation": "Astra: 10/50, cached 1, batch 5/25 USD per million; Luna: .10/.50, cached .01, batch .05/.25. Tool billing is additional.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "latency",
      "url": "https://developers.openai.com/api/docs/guides/latency-optimization",
      "title": "Latency optimization",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Generate fewer tokens; Use fewer input tokens; Make users wait less",
      "observation": "Heuristics: half the output may halve latency; half ordinary-context input may improve 1–5%. Streaming exposes output sooner.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "cache",
      "url": "https://developers.openai.com/api/docs/guides/prompt-caching",
      "title": "Prompt caching",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Rendered prefix; How caching works; Gotchas",
      "observation": "Rendered-prefix equality, eligible boundaries and model-specific minimums govern reuse; common text alone need not hit cache.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "claude",
      "url": "https://claude.com/pricing",
      "title": "Pricing",
      "author_or_maintainer": "Anthropic",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Latest models: Fable 5.1 and Haiku 5.5",
      "observation": "Fable 10/50 and .25 cache read. Haiku .10/.50 and .01 cache read apply through 100K prompt tokens; longer prompts cost .50/2.50 and .05 cache read.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "claudedetail",
      "url": "https://platform.claude.com/docs/en/about-claude/pricing",
      "title": "Pricing",
      "author_or_maintainer": "Anthropic",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Batch processing; Long context pricing",
      "observation": "Batch input/output discount is 50%; Haiku 5.5 has a 100K prompt-length price boundary.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "gemini",
      "url": "https://ai.google.dev/gemini-api/docs/pricing",
      "title": "Gemini Developer API pricing",
      "author_or_maintainer": "Google",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Gemini 4 Argon Standard and Batch tables",
      "observation": "Standard 2/10 and cache .10 through 30 November 2026; 4/20 and .20 from 1 December. Batch 1/5 then 2/10; implicit caching only.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "mmlu",
      "url": "https://arxiv.org/abs/2009.03300",
      "title": "Measuring Massive Multitask Language Understanding",
      "author_or_maintainer": "Dan Hendrycks et al.",
      "original_year_or_first_arxiv_year": "2020 (ICLR 2021)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; author list; submission history",
      "observation": "57 academic/professional tasks; the historical models lag expert accuracy. Both 2020 preprint and 2021 venue dating are legitimate.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "helm",
      "url": "https://arxiv.org/abs/2211.09110",
      "title": "Holistic Evaluation of Language Models",
      "author_or_maintainer": "Percy Liang et al.",
      "original_year_or_first_arxiv_year": "2022",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2; author list",
      "observation": "Seven metrics, 16 core and 26 targeted scenarios, 30 models; core coverage moves from 17.9% to 96.0%.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "yang",
      "url": "https://arxiv.org/abs/2311.04850",
      "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "author_or_maintainer": "Shuo Yang et al.",
      "original_year_or_first_arxiv_year": "2023",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2; contamination construction and decontamination method",
      "observation": "Paraphrase/translation evades n-gram filtering; reported HumanEval overlap 8–18%; injected variants let a 13B model overfit. Not a universal original-versus-rewrite performance audit.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "xiao",
      "url": "https://arxiv.org/abs/2609.02899",
      "title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards",
      "author_or_maintainer": "Xingyao Xiao and Yihong Cheng",
      "original_year_or_first_arxiv_year": "2026",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract and author list",
      "observation": "Paraphrase-controlled study of selected models and four benchmarks reports score inflation with little ranking movement; preprint scope matters.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "zhengpro",
      "url": "https://arxiv.org/abs/2609.08149",
      "title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents",
      "author_or_maintainer": "Pujun Zheng et al.",
      "original_year_or_first_arxiv_year": "2026",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v3, revised 9 October 2026",
      "observation": "Reports solution/hidden-information leakage, reward hacking and task-quality defects in the studied SWE-Bench Pro version.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "position",
      "url": "https://arxiv.org/abs/2406.07791",
      "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
      "author_or_maintainer": "Lin Shi et al.",
      "original_year_or_first_arxiv_year": "2024 (revised 2025)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; author list; submission history",
      "observation": "Order bias varies by judge and task. This paper is by Shi and colleagues, not Zheng and colleagues.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "variance",
      "url": "https://arxiv.org/abs/2103.03098",
      "title": "Accounting for Variance in Machine Learning Benchmarks",
      "author_or_maintainer": "Xavier Bouthillier et al.",
      "original_year_or_first_arxiv_year": "2021",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; comparison-method discussion",
      "observation": "Data, initialisation and tuning variation affect comparisons; repeated trials and variance-aware comparison are justified.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "swe",
      "url": "https://arxiv.org/abs/2310.06770",
      "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "author_or_maintainer": "Carlos E. Jimenez et al.",
      "original_year_or_first_arxiv_year": "2023 (ICLR 2024)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v3; task/evaluation definition",
      "observation": "2,294 issues across 12 Python repositories; historical best Claude 2 resolves 1.96%; code changes judged using repository tests.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "swesite",
      "url": "https://www.swebench.com/",
      "title": "SWE-bench Leaderboards",
      "author_or_maintainer": "SWE-bench team",
      "original_year_or_first_arxiv_year": "living maintainer page",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Variant descriptions, not rank rows",
      "observation": "Verified/Bash Only: 500, common mini-SWE-agent; Lite: 300; Multilingual: 300 across 42 repositories and nine languages.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "code",
      "url": "https://arxiv.org/abs/2107.03374",
      "title": "Evaluating Large Language Models Trained on Code",
      "author_or_maintainer": "Mark Chen et al.",
      "original_year_or_first_arxiv_year": "2021",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; HTML v2 section 2.1 and Appendix A",
      "observation": "HumanEval checks function correctness from docstrings. Per-task pass@k estimator is 1−C(n−c,k)/C(n,k), n≥k; average across tasks.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "tau",
      "url": "https://arxiv.org/abs/2406.12045",
      "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "author_or_maintainer": "Shunyu Yao et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; HTML v1 section 3",
      "observation": "Simulated users, API policies and final database state. pass^k averages all-k-success probability; estimator C(c,k)/C(n,k). Historical success <50%, retail pass^8 <25%.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "judge",
      "url": "https://arxiv.org/abs/2306.05685",
      "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "author_or_maintainer": "Lianmin Zheng et al.",
      "original_year_or_first_arxiv_year": "2023",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v4; HTML sections 3.3–3.4",
      "observation": "Historical >80% human preference agreement. Position/verbosity effects; self-enhancement not established conclusively. Conservative swap rule requires agreement, otherwise tie; mitigation does not ensure accuracy.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "big",
      "url": "https://arxiv.org/abs/2206.04615",
      "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "author_or_maintainer": "Aarohi Srivastava et al.",
      "original_year_or_first_arxiv_year": "2022",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v3; bibliographic record",
      "observation": "204 tasks, 450 contributors, 132 institutions, human expert baselines; scale improves performance/calibration but absolute results remain poor; brittle metrics affect apparent breakthroughs.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "arena",
      "url": "https://arxiv.org/abs/2403.04132",
      "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
      "author_or_maintainer": "Wei-Lin Chiang et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract and author list",
      "observation": "Pairwise crowdsourced preference; over 240K historical votes; authors report agreement with experts.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "illusion",
      "url": "https://arxiv.org/abs/2504.20879",
      "title": "The Leaderboard Illusion",
      "author_or_maintainer": "Shivalika Singh et al.",
      "original_year_or_first_arxiv_year": "2025",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2; author list",
      "observation": "Private variants/selective disclosure and sampling asymmetry; estimates 19.2%, 20.4% versus 29.7% for 83 open-weight models; Arena distribution gains are study-specific.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "better",
      "url": "https://arxiv.org/abs/2411.12990",
      "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "author_or_maintainer": "Anka Reuel et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; NeurIPS acceptance note",
      "observation": "46 lifecycle practices applied to 24 benchmarks; quality varies, with significance and replication reporting gaps.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "kapoor",
      "url": "https://arxiv.org/abs/2407.01502",
      "title": "AI Agents That Matter",
      "author_or_maintainer": "Sayash Kapoor et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract",
      "observation": "Joint cost/accuracy optimisation, inadequate holdouts, model/downstream needs and non-standardised reporting are the paper’s stated concerns.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "gaia",
      "url": "https://arxiv.org/abs/2311.12983",
      "title": "GAIA: a benchmark for General AI Assistants",
      "author_or_maintainer": "Grégoire Mialon et al.",
      "original_year_or_first_arxiv_year": "2023",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract; author list",
      "observation": "466 questions, 300 answers retained; historical humans 92%, GPT-4 with plugins 15%.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "live",
      "url": "https://arxiv.org/abs/2403.07974",
      "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
      "author_or_maintainer": "Naman Jain et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstracts v1 and v2; submission history",
      "observation": "400 problems in both abstracts; v1 ends February 2024, v2 ends May 2024. Calling the May window the initial release conflates versions.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "deng",
      "url": "https://arxiv.org/abs/2311.09783",
      "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
      "author_or_maintainer": "Chunyuan Deng et al.",
      "original_year_or_first_arxiv_year": "2023 (NAACL 2024)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2",
      "observation": "TS-Guessing masks wrong options; MMLU exact-match guessing 52% ChatGPT, 57% GPT-4; this is not a no-contamination certificate.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "cheng",
      "url": "https://arxiv.org/abs/2502.14425",
      "title": "A Survey on Data Contamination for Large Language Models",
      "author_or_maintainer": "Yuxing Cheng, Yi Chang and Yuan Wu",
      "original_year_or_first_arxiv_year": "2025",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2",
      "observation": "White/grey/black-box detection; updating, rewriting and prevention strategies. A survey is secondary synthesis, not an original experiment.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "miller",
      "url": "https://arxiv.org/abs/2411.00640",
      "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "author_or_maintainer": "Evan Miller",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Full HTML v1 sections 2–5",
      "observation": "CLT, paired/clustered errors and power. Example: 969 questions for .03 effect. Uniform-difficulty example gives 1/3, 1/2 and 2/3 variance reductions. Cluster ratios reach 3.05.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "madaan",
      "url": "https://arxiv.org/abs/2406.10229",
      "title": "Quantifying Variance in Evaluation Benchmarks",
      "author_or_maintainer": "Lovish Madaan et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract",
      "observation": "Studies initialisation/training variance and choice-versus-completion formatting, especially smaller models.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "card",
      "url": "https://arxiv.org/abs/2010.06595",
      "title": "With Little Power Comes Great Responsibility",
      "author_or_maintainer": "Dallas Card et al.",
      "original_year_or_first_arxiv_year": "2020",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract",
      "observation": "Underpowered NLP comparisons; 2,000-sentence MT tests approximately 75% power for one BLEU under stated assumptions.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "shankar",
      "url": "https://arxiv.org/abs/2404.12272",
      "title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
      "author_or_maintainer": "Shreya Shankar et al.",
      "original_year_or_first_arxiv_year": "2024",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract",
      "observation": "Criteria evolve through seeing outputs; evaluator validation is iterative.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "shi",
      "url": "https://arxiv.org/abs/2310.16789",
      "title": "Detecting Pretraining Data from Large Language Models",
      "author_or_maintainer": "Weijia Shi et al.",
      "original_year_or_first_arxiv_year": "2023",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v3",
      "observation": "Studies pretraining exposure using black-box access without the pretraining corpus; method-specific assumptions remain.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "datasheet",
      "url": "https://arxiv.org/abs/1803.09010",
      "title": "Datasheets for Datasets",
      "author_or_maintainer": "Timnit Gebru et al.",
      "original_year_or_first_arxiv_year": "2018 (CACM 2021)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v8",
      "observation": "Dataset documentation covers motivation, composition, collection and intended uses.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "oren",
      "url": "https://arxiv.org/abs/2310.17623",
      "title": "Proving Test Set Contamination in Black Box Language Models",
      "author_or_maintainer": "Yonatan Oren et al.",
      "original_year_or_first_arxiv_year": "2023",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2",
      "observation": "Canonical-versus-shuffled likelihood test relies on exchangeability; proves some contamination under assumptions, not universal cleanliness.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "sclar",
      "url": "https://arxiv.org/abs/2310.11324",
      "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
      "author_or_maintainer": "Melanie Sclar et al.",
      "original_year_or_first_arxiv_year": "2023 (ICLR 2024)",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Abstract v2",
      "observation": "Meaning-preserving formats affect studied few-shot models; preferred formats correlate weakly across models.",
      "historical_limit": "Paper versions available before the submission research date; historical results are not current-model estimates."
    },
    {
      "source_id": "checklist",
      "url": "https://aclanthology.org/2020.acl-main.442/",
      "title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
      "author_or_maintainer": "Marco Tulio Ribeiro et al.",
      "original_year_or_first_arxiv_year": "2020",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "ACL abstract and citation record",
      "observation": "Behavioural-test matrix joins capabilities and test types; general accuracy misses specific failures.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "leak",
      "url": "https://scikit-learn.org/stable/common_pitfalls.html",
      "title": "Common pitfalls and recommended practices",
      "author_or_maintainer": "scikit-learn developers",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Data leakage, section 12.2",
      "observation": "Unavailable-at-prediction information and test-influenced choices bias scores. Split before fitting transformations.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "metrics",
      "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
      "title": "precision_recall_fscore_support",
      "author_or_maintainer": "scikit-learn developers",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Definitions and average parameter",
      "observation": "Precision TP/(TP+FP), recall TP/(TP+FN); macro unweighted, weighted by class support; micro pools counts.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "kappa",
      "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.cohen_kappa_score.html",
      "title": "cohen_kappa_score",
      "author_or_maintainer": "scikit-learn developers",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Definition; weights parameter",
      "observation": "Kappa=(observed−expected)/(1−expected); expected uses annotator marginals. Linear/quadratic weighting supports ordinal disagreements.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "bootstrap",
      "url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
      "title": "scipy.stats.bootstrap",
      "author_or_maintainer": "SciPy developers",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Introduction; paired parameter; confidence interpretation",
      "observation": "Paired resampling preserves common indices; percentile interval and repeated-sampling coverage; degenerate and sparse cases need care.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "wilson",
      "url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm",
      "title": "7.2.4.1. Confidence intervals",
      "author_or_maintainer": "NIST/SEMATECH",
      "original_year_or_first_arxiv_year": "living reference",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Wilson formula",
      "observation": "Wilson interval inverts the binomial score test.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "harness",
      "url": "https://github.com/EleutherAI/lm-evaluation-harness",
      "title": "Language Model Evaluation Harness",
      "author_or_maintainer": "EleutherAI",
      "original_year_or_first_arxiv_year": "living repository",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "README description and basic task/model arguments",
      "observation": "Shared few-shot evaluation framework; task and model configuration are explicit inputs.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "inspect",
      "url": "https://inspect.aisi.org.uk/",
      "title": "Inspect",
      "author_or_maintainer": "UK AI Security Institute and Meridian Labs",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Hello Inspect; task and sandbox examples",
      "observation": "Tasks compose datasets, solvers and scorers; agents/tools and Docker sandboxes are supported. Homepage does not establish citation year 2024.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "groups",
      "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
      "title": "Cross-validation: evaluating estimator performance",
      "author_or_maintainer": "scikit-learn developers",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Grouped and time-series data sections",
      "observation": "Independent groups stay in separate folds; chronological evaluation addresses ordered data. Group choice follows the generalisation target.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    },
    {
      "source_id": "workflow",
      "url": "https://learn.chatgpt.com/docs/long-running-work",
      "title": "Long-running work",
      "author_or_maintainer": "OpenAI",
      "original_year_or_first_arxiv_year": "living documentation",
      "retrieved_utc_date": "2026-10-25",
      "inspected_location": "Page identity; not used for topical fact claims",
      "observation": "Workflow-only reference in submissions; not evidence of benchmark performance.",
      "historical_limit": "Live retrieval corroborates the cited passage; no archived snapshot of the candidate research day was obtained."
    }
  ],
  "claim_records": [
    {
      "claim_id": "A-model-benchmarking-01",
      "cluster": "A",
      "article_id": "model-benchmarking",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "location": {
        "line": 33,
        "excerpt": "MMLU, for example, was introduced as a 57-subject"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "mmlu",
          "title": "Measuring Massive Multitask Language Understanding",
          "url": "https://arxiv.org/abs/2009.03300",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "57-task description and historical scope agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-model-benchmarking-02",
      "cluster": "A",
      "article_id": "model-benchmarking",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "location": {
        "line": 33,
        "excerpt": "HELM makes the broader point"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "helm",
          "title": "Holistic Evaluation of Language Models",
          "url": "https://arxiv.org/abs/2211.09110",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Multi-scenario/multi-metric description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-model-benchmarking-03",
      "cluster": "A",
      "article_id": "model-benchmarking",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "location": {
        "line": 76,
        "excerpt": "costs can include multiple token categories and tool calls"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Token categories and separate tool charges confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-model-benchmarking-04",
      "cluster": "A",
      "article_id": "model-benchmarking",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "location": {
        "line": 84,
        "excerpt": "Benchmarks such as SWE-bench were created"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swe",
          "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
          "url": "https://arxiv.org/abs/2310.06770",
          "inspected_location": "Abstract v3; task/evaluation definition"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Repository-plus-issue task description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-model-benchmarking-05",
      "cluster": "A",
      "article_id": "model-benchmarking",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "location": {
        "line": 70,
        "excerpt": "A statistical interval does not fix biased sampling"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Inference remains conditional on sampling design.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-public-benchmarks-01",
      "cluster": "A",
      "article_id": "public-benchmarks",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "location": {
        "line": 21,
        "excerpt": "MMLU, for example, was introduced as a 57-subject"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "mmlu",
          "title": "Measuring Massive Multitask Language Understanding",
          "url": "https://arxiv.org/abs/2009.03300",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "57-task academic/professional scope agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-public-benchmarks-02",
      "cluster": "A",
      "article_id": "public-benchmarks",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "location": {
        "line": 31,
        "excerpt": "HELM’s original work emphasised"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "helm",
          "title": "Holistic Evaluation of Language Models",
          "url": "https://arxiv.org/abs/2211.09110",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Scenario and metric matrix supports the claim.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-public-benchmarks-03",
      "cluster": "A",
      "article_id": "public-benchmarks",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "location": {
        "line": 39,
        "excerpt": "Zheng et al., 2024"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "position",
          "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
          "url": "https://arxiv.org/abs/2406.07791",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Linked study exists and supports order bias, but its first author is Lin Shi. 2024 preprint dating is valid.",
      "verdict": "wrong_bibliographic_metadata",
      "severity": "minor",
      "repair": "Use Shi et al., 2024, optionally specify the 2025 revision.",
      "publication_blocker": false
    },
    {
      "claim_id": "A-public-benchmarks-04",
      "cluster": "A",
      "article_id": "public-benchmarks",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "location": {
        "line": 45,
        "excerpt": "Research has used rephrased examples to probe"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "yang",
          "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
          "url": "https://arxiv.org/abs/2311.04850",
          "inspected_location": "Abstract v2; contamination construction and decontamination method"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Broad contamination rationale supported; experimental rewrites here inject contamination and motivate semantic filtering, not a general performance-drop detector.",
      "verdict": "partially_supported",
      "severity": "minor",
      "repair": "Describe paraphrase-based contamination and semantic decontamination precisely.",
      "publication_blocker": false
    },
    {
      "claim_id": "A-task-evaluation-01",
      "cluster": "A",
      "article_id": "task-evaluation",
      "article_file": "A/articles/build-task-eval.md",
      "location": {
        "line": 37,
        "excerpt": "group them so a random split does not put almost identical cases on both sides"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Group separation is appropriate to unseen-group generalisation.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-task-evaluation-02",
      "cluster": "A",
      "article_id": "task-evaluation",
      "article_file": "A/articles/build-task-eval.md",
      "location": {
        "line": 61,
        "excerpt": "A pilot is not a result to promote as a final estimate"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Post-inspection tuning changes the evidence role.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-task-evaluation-03",
      "cluster": "A",
      "article_id": "task-evaluation",
      "article_file": "A/articles/build-task-eval.md",
      "location": {
        "line": 35,
        "excerpt": "Synthetic examples are useful for controlled coverage"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "checklist",
          "title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
          "url": "https://aclanthology.org/2020.acl-main.442/",
          "inspected_location": "ACL abstract and citation record"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Designed challenges test behaviours; they do not estimate prevalence.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-dataset-integrity-01",
      "cluster": "A",
      "article_id": "dataset-integrity",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "location": {
        "line": 21,
        "excerpt": "Group records by the unit that could leak"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Grouping matches the proposed generalisation question.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-dataset-integrity-02",
      "cluster": "A",
      "article_id": "dataset-integrity",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "location": {
        "line": 31,
        "excerpt": "comparing original benchmark items with semantically equivalent rewrites"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "yang",
          "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
          "url": "https://arxiv.org/abs/2311.04850",
          "inspected_location": "Abstract v2; contamination construction and decontamination method"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Study supports paraphrase contamination, but this description conflates injected variants/semantic filtering with a performance-comparison audit.",
      "verdict": "partially_supported",
      "severity": "minor",
      "repair": "Distinguish Yang’s contamination construction from a paraphrase-controlled score audit.",
      "publication_blocker": false
    },
    {
      "claim_id": "A-dataset-integrity-03",
      "cluster": "A",
      "article_id": "dataset-integrity",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "location": {
        "line": 31,
        "excerpt": "Xiao and Cheng, 2026"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "xiao",
          "title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards",
          "url": "https://arxiv.org/abs/2609.02899",
          "inspected_location": "Abstract and author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Authors/year/title match; narrow leaderboard conclusion faithfully qualified.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-dataset-integrity-04",
      "cluster": "A",
      "article_id": "dataset-integrity",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "location": {
        "line": 35,
        "excerpt": "A holdout loses value when its answers influence development"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Test-influenced model choices bias final evidence.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-metrics-01",
      "cluster": "A",
      "article_id": "evaluation-metrics",
      "article_file": "A/articles/metrics-rubrics.md",
      "location": {
        "line": 21,
        "excerpt": "Precision answers how often predicted positives are correct"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Precision and recall definitions agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-metrics-02",
      "cluster": "A",
      "article_id": "evaluation-metrics",
      "article_file": "A/articles/metrics-rubrics.md",
      "location": {
        "line": 53,
        "excerpt": "A macro average gives each class equal weight"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Macro and support-weighted distinction agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-metrics-03",
      "cluster": "A",
      "article_id": "evaluation-metrics",
      "article_file": "A/articles/metrics-rubrics.md",
      "location": {
        "line": 31,
        "excerpt": "recommends task-specific evaluation tied to intended behaviour"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Task-specific evaluation recommendation is present.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-metrics-04",
      "cluster": "A",
      "article_id": "evaluation-metrics",
      "article_file": "A/articles/metrics-rubrics.md",
      "location": {
        "line": 45,
        "excerpt": "testing a model grader against examples with known quality ordering"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "graders",
          "title": "Graders",
          "url": "https://developers.openai.com/api/docs/guides/graders",
          "inspected_location": "Grader types; Model graders; Grader hacking"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Expert ordering check is explicitly documented.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-human-judging-01",
      "cluster": "A",
      "article_id": "human-judging",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "location": {
        "line": 35,
        "excerpt": "Zheng et al., 2024"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "position",
          "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
          "url": "https://arxiv.org/abs/2406.07791",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Order-bias study supports the substance; cited author label is wrong.",
      "verdict": "wrong_bibliographic_metadata",
      "severity": "minor",
      "repair": "Change to Shi et al.; retain valid preprint year.",
      "publication_blocker": false
    },
    {
      "claim_id": "A-human-judging-02",
      "cluster": "A",
      "article_id": "human-judging",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "location": {
        "line": 51,
        "excerpt": "calibration of model-based judges against human feedback"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Human calibration recommendation is present.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-human-judging-03",
      "cluster": "A",
      "article_id": "human-judging",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "location": {
        "line": 27,
        "excerpt": "consensus does not mean the original disagreement never existed"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "shankar",
          "title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
          "url": "https://arxiv.org/abs/2404.12272",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Preserving initial criteria/ratings is sound process advice.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-human-judging-04",
      "cluster": "A",
      "article_id": "human-judging",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "location": {
        "line": 71,
        "excerpt": "it may learn to produce features the judge likes"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "graders",
          "title": "Graders",
          "url": "https://developers.openai.com/api/docs/guides/graders",
          "inspected_location": "Grader types; Model graders; Grader hacking"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Grader hacking is explicitly discussed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-repeatability-01",
      "cluster": "A",
      "article_id": "repeatability",
      "article_file": "A/articles/repeatability-uncertainty.md",
      "location": {
        "line": 44,
        "excerpt": "Bouthillier et al., 2021"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "variance",
          "title": "Accounting for Variance in Machine Learning Benchmarks",
          "url": "https://arxiv.org/abs/2103.03098",
          "inspected_location": "Abstract; comparison-method discussion"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Authors/year/title match; variance-aware comparison supported.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-repeatability-02",
      "cluster": "A",
      "article_id": "repeatability",
      "article_file": "A/articles/repeatability-uncertainty.md",
      "location": {
        "line": 42,
        "excerpt": "resample conversations if conversations contain multiple related turns"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "bootstrap",
          "title": "scipy.stats.bootstrap",
          "url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
          "inspected_location": "Introduction; paired parameter; confidence interpretation"
        },
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Bootstrap unit must preserve dependence; application is reviewer-judged.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-repeatability-03",
      "cluster": "A",
      "article_id": "repeatability",
      "article_file": "A/articles/repeatability-uncertainty.md",
      "location": {
        "line": 30,
        "excerpt": "Do not treat all 300 as independent examples"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Repeated outputs share cases; no extra independent tasks.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-repeatability-04",
      "cluster": "A",
      "article_id": "repeatability",
      "article_file": "A/articles/repeatability-uncertainty.md",
      "location": {
        "line": 46,
        "excerpt": "If two intervals overlap, that alone is not a formal test"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Analyse the paired difference rather than marginal interval overlap.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-fair-comparison-01",
      "cluster": "A",
      "article_id": "fair-comparison",
      "article_file": "A/articles/fair-model-comparisons.md",
      "location": {
        "line": 27,
        "excerpt": "do not assume identical text creates identical conditions"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "sclar",
          "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
          "url": "https://arxiv.org/abs/2310.11324",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Studied format sensitivity supports the qualification.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-fair-comparison-02",
      "cluster": "A",
      "article_id": "fair-comparison",
      "article_file": "A/articles/fair-model-comparisons.md",
      "location": {
        "line": 33,
        "excerpt": "reserve a final holdout for the decision"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Test-based tuning would bias comparison.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-fair-comparison-03",
      "cluster": "A",
      "article_id": "fair-comparison",
      "article_file": "A/articles/fair-model-comparisons.md",
      "location": {
        "line": 45,
        "excerpt": "A provider-specific constrained-output feature may be a legitimate advantage"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "harness",
          "title": "Language Model Evaluation Harness",
          "url": "https://github.com/EleutherAI/lm-evaluation-harness",
          "inspected_location": "README description and basic task/model arguments"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Boundary disclosure is reasonable; common harness alone guarantees no attribution.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-latency-cost-01",
      "cluster": "A",
      "article_id": "latency-cost",
      "article_file": "A/articles/latency-cost.md",
      "location": {
        "line": 29,
        "excerpt": "distinguishes token categories and lists separate prices"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Categories and separate tool units confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-latency-cost-02",
      "cluster": "A",
      "article_id": "latency-cost",
      "article_file": "A/articles/latency-cost.md",
      "location": {
        "line": 37,
        "excerpt": "time to first token may matter, but the first token can be an unhelpful preamble"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "latency",
          "title": "Latency optimization",
          "url": "https://developers.openai.com/api/docs/guides/latency-optimization",
          "inspected_location": "Generate fewer tokens; Use fewer input tokens; Make users wait less"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Streaming start is distinct from usable completion.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-latency-cost-03",
      "cluster": "A",
      "article_id": "latency-cost",
      "article_file": "A/articles/latency-cost.md",
      "location": {
        "line": 57,
        "excerpt": "A’s total is $258 and B’s is $127"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Independent arithmetic: 18+60×4=258; 27+25×4=127. Hypothetical, not measured.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-latency-cost-04",
      "cluster": "A",
      "article_id": "latency-cost",
      "article_file": "A/articles/latency-cost.md",
      "location": {
        "line": 27,
        "excerpt": "input charges + output charges + tool charges + retry charges"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Bookkeeping requires initial charges to exclude retry usage; otherwise retries are counted twice. No numeric double count occurs here.",
      "verdict": "needs_clarification",
      "severity": "minor",
      "repair": "Define totals over all attempts and avoid a second retry charge.",
      "publication_blocker": false
    },
    {
      "claim_id": "A-agent-evaluation-01",
      "cluster": "A",
      "article_id": "agent-evaluation",
      "article_file": "A/articles/agent-system-evaluation.md",
      "location": {
        "line": 33,
        "excerpt": "trace grading as a way to assess tool selection, handoffs"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "agentdocs",
          "title": "Evaluate agent workflows",
          "url": "https://developers.openai.com/api/docs/guides/agent-evals",
          "inspected_location": "Start with traces; Move to datasets and eval runs"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Workflow trace questions match.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-agent-evaluation-02",
      "cluster": "A",
      "article_id": "agent-evaluation",
      "article_file": "A/articles/agent-system-evaluation.md",
      "location": {
        "line": 43,
        "excerpt": "Zheng et al., 2026"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "zhengpro",
          "title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents",
          "url": "https://arxiv.org/abs/2609.08149",
          "inspected_location": "Abstract v3, revised 9 October 2026"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Correct study, authors/year, and narrow leakage/task-quality caveat.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-agent-evaluation-03",
      "cluster": "A",
      "article_id": "agent-evaluation",
      "article_file": "A/articles/agent-system-evaluation.md",
      "location": {
        "line": 21,
        "excerpt": "If tool responses are mocked"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "inspect",
          "title": "Inspect",
          "url": "https://inspect.aisi.org.uk/",
          "inspected_location": "Hello Inspect; task and sandbox examples"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Fixtures isolate behaviour; live-failure generalisation remains limited.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-agent-evaluation-04",
      "cluster": "A",
      "article_id": "agent-evaluation",
      "article_file": "A/articles/agent-system-evaluation.md",
      "location": {
        "line": 61,
        "excerpt": "attribution requires an ablation"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "harness",
          "title": "Language Model Evaluation Harness",
          "url": "https://github.com/EleutherAI/lm-evaluation-harness",
          "inspected_location": "README description and basic task/model arguments"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Controlled component changes clarify attribution.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-maintenance-01",
      "cluster": "A",
      "article_id": "evaluation-maintenance",
      "article_file": "A/articles/maintaining-evaluations.md",
      "location": {
        "line": 25,
        "excerpt": "continuous evaluation on changes and growing a suite"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Continuous evaluation and new-case guidance confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-maintenance-02",
      "cluster": "A",
      "article_id": "evaluation-maintenance",
      "article_file": "A/articles/maintaining-evaluations.md",
      "location": {
        "line": 46,
        "excerpt": "When a judge model or prompt changes, rerun a calibration set"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "graders",
          "title": "Graders",
          "url": "https://developers.openai.com/api/docs/guides/graders",
          "inspected_location": "Grader types; Model graders; Grader hacking"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Revalidation follows grader-quality checks.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "A-evaluation-maintenance-03",
      "cluster": "A",
      "article_id": "evaluation-maintenance",
      "article_file": "A/articles/maintaining-evaluations.md",
      "location": {
        "line": 38,
        "excerpt": "Report anchor and new-set scores separately"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Different samples need explicit comparison boundaries.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-P01-01",
      "cluster": "B",
      "article_id": "P01",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "location": {
        "line": 33,
        "excerpt": "test spanning 57 subjects"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "mmlu",
          "title": "Measuring Massive Multitask Language Understanding",
          "url": "https://arxiv.org/abs/2009.03300",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "57-task scope agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-P01-02",
      "cluster": "B",
      "article_id": "P01",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "location": {
        "line": 73,
        "excerpt": "calibration of automated scoring with human feedback"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Calibration recommendation confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-P01-03",
      "cluster": "B",
      "article_id": "P01",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "location": {
        "line": 105,
        "excerpt": "Its cost per accepted outcome is $0.15"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Independent arithmetic: 12/80=.15; invented totals are labelled.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-P01-04",
      "cluster": "B",
      "article_id": "P01",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "location": {
        "line": 63,
        "excerpt": "Split related items together"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Appropriate group boundary for unseen-group inference.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S01-01",
      "cluster": "B",
      "article_id": "S01",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "location": {
        "line": 23,
        "excerpt": "functional correctness of programs generated from docstrings"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "code",
          "title": "Evaluating Large Language Models Trained on Code",
          "url": "https://arxiv.org/abs/2107.03374",
          "inspected_location": "Abstract; HTML v2 section 2.1 and Appendix A"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "HumanEval task boundary agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S01-02",
      "cluster": "B",
      "article_id": "S01",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "location": {
        "line": 35,
        "excerpt": "at least one successful sample among several attempts"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "code",
          "title": "Evaluating Large Language Models Trained on Code",
          "url": "https://arxiv.org/abs/2107.03374",
          "inspected_location": "Abstract; HTML v2 section 2.1 and Appendix A"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "pass@k interpretation agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S01-03",
      "cluster": "B",
      "article_id": "S01",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "location": {
        "line": 43,
        "excerpt": "default Verified “Bash Only” view"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swesite",
          "title": "SWE-bench Leaderboards",
          "url": "https://www.swebench.com/",
          "inspected_location": "Variant descriptions, not rank rows"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Current maintainer description corroborates the dated claim.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S01-04",
      "cluster": "B",
      "article_id": "S01",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "location": {
        "line": 53,
        "excerpt": "accuracy, robustness and efficiency"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "helm",
          "title": "Holistic Evaluation of Language Models",
          "url": "https://arxiv.org/abs/2211.09110",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "HELM metric scope agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S02-01",
      "cluster": "B",
      "article_id": "S02",
      "article_file": "B/articles/design-llm-evaluation-tasks.md",
      "location": {
        "line": 55,
        "excerpt": "CheckList organises tests around linguistic capabilities and test types"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "checklist",
          "title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
          "url": "https://aclanthology.org/2020.acl-main.442/",
          "inspected_location": "ACL abstract and citation record"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Matrix description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S02-02",
      "cluster": "B",
      "article_id": "S02",
      "article_file": "B/articles/design-llm-evaluation-tasks.md",
      "location": {
        "line": 33,
        "excerpt": "giving the model the entire final conversation can reveal information"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Prediction-time information boundary supports the warning.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S02-03",
      "cluster": "B",
      "article_id": "S02",
      "article_file": "B/articles/design-llm-evaluation-tasks.md",
      "location": {
        "line": 65,
        "excerpt": "Keep the pair in the same data split"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Grouping scenario variants limits overlap leakage.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S03-01",
      "cluster": "B",
      "article_id": "S03",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "location": {
        "line": 21,
        "excerpt": "splitting before fitting data transformations"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Split-before-fit instruction confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S03-02",
      "cluster": "B",
      "article_id": "S03",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "location": {
        "line": 23,
        "excerpt": "pretraining-data detection explicitly studies that limited-access problem"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "shi",
          "title": "Detecting Pretraining Data from Large Language Models",
          "url": "https://arxiv.org/abs/2310.16789",
          "inspected_location": "Abstract v3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Black-box/no-corpus problem definition agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S03-03",
      "cluster": "B",
      "article_id": "S03",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "location": {
        "line": 31,
        "excerpt": "Datasheets for Datasets proposes this kind of documentation"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "datasheet",
          "title": "Datasheets for Datasets",
          "url": "https://arxiv.org/abs/1803.09010",
          "inspected_location": "Abstract v8"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Documentation fields and purpose agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S03-04",
      "cluster": "B",
      "article_id": "S03",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "location": {
        "line": 65,
        "excerpt": "canonical versus shuffled benchmark ordering"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "oren",
          "title": "Proving Test Set Contamination in Black Box Language Models",
          "url": "https://arxiv.org/abs/2310.17623",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Exchangeability and likelihood-test description agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S04-01",
      "cluster": "B",
      "article_id": "S04",
      "article_file": "B/articles/llm-evaluation-metrics-rubrics.md",
      "location": {
        "line": 41,
        "excerpt": "Precision for escalation is 15 divided by 25"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "TP15 FP10 FN5 TN70 → precision .60, recall .75, F1 .6667, accuracy .85.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S04-02",
      "cluster": "B",
      "article_id": "S04",
      "article_file": "B/articles/llm-evaluation-metrics-rubrics.md",
      "location": {
        "line": 47,
        "excerpt": "A macro average gives classes equal weight"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Macro versus support weighting agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S04-03",
      "cluster": "B",
      "article_id": "S04",
      "article_file": "B/articles/llm-evaluation-metrics-rubrics.md",
      "location": {
        "line": 25,
        "excerpt": "A finite test suite provides evidence for its cases"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "code",
          "title": "Evaluating Large Language Models Trained on Code",
          "url": "https://arxiv.org/abs/2107.03374",
          "inspected_location": "Abstract; HTML v2 section 2.1 and Appendix A"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Functional tests support covered cases, not universal correctness.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S05-01",
      "cluster": "B",
      "article_id": "S05",
      "article_file": "B/articles/human-reviewers-llm-judges.md",
      "location": {
        "line": 49,
        "excerpt": "position, verbosity and self-enhancement biases"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "judge",
          "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
          "url": "https://arxiv.org/abs/2306.05685",
          "inspected_location": "Abstract v4; HTML sections 3.3–3.4"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Accurately reports topics studied without asserting all are proven universal.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S05-02",
      "cluster": "B",
      "article_id": "S05",
      "article_file": "B/articles/human-reviewers-llm-judges.md",
      "location": {
        "line": 61,
        "excerpt": "A judge that always passes achieves 90% agreement"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Independent arithmetic: 90/100=.90, zero detected failures.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S05-03",
      "cluster": "B",
      "article_id": "S05",
      "article_file": "B/articles/human-reviewers-llm-judges.md",
      "location": {
        "line": 51,
        "excerpt": "test a sample in both orders"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "position",
          "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
          "url": "https://arxiv.org/abs/2406.07791",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Order-sensitivity diagnostic is supported.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S05-04",
      "cluster": "B",
      "article_id": "S05",
      "article_file": "B/articles/human-reviewers-llm-judges.md",
      "location": {
        "line": 57,
        "excerpt": "instruction text alone is not proof of resistance"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "graders",
          "title": "Graders",
          "url": "https://developers.openai.com/api/docs/guides/graders",
          "inspected_location": "Grader types; Model graders; Grader hacking"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Adversarial grader checks are appropriate; no protection guarantee.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S06-01",
      "cluster": "B",
      "article_id": "S06",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "location": {
        "line": 39,
        "excerpt": "A 95% Wilson interval is approximately 71.1% to 86.7%"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "wilson",
          "title": "7.2.4.1. Confidence intervals",
          "url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm",
          "inspected_location": "Wilson formula"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Recalculated with z=1.96: .71117–.86663.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S06-02",
      "cluster": "B",
      "article_id": "S06",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "location": {
        "line": 43,
        "excerpt": "long-run coverage of the interval procedure"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "bootstrap",
          "title": "scipy.stats.bootstrap",
          "url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
          "inspected_location": "Introduction; paired parameter; confidence interpretation"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Frequentist repeated-sampling interpretation agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S06-03",
      "cluster": "B",
      "article_id": "S06",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "location": {
        "line": 51,
        "excerpt": "resamples corresponding observations together"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "bootstrap",
          "title": "scipy.stats.bootstrap",
          "url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
          "inspected_location": "Introduction; paired parameter; confidence interpretation"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "paired=True uses common sampled indices.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S06-04",
      "cluster": "B",
      "article_id": "S06",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "location": {
        "line": 31,
        "excerpt": "A therefore passes 80 and B passes 85"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "75+5=80; 75+10=85; paired lead .05. Hypothetical.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S06-05",
      "cluster": "B",
      "article_id": "S06",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "location": {
        "line": 71,
        "excerpt": "absence of a statistically significant difference does not establish equivalence"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Null non-rejection does not demonstrate a margin.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S07-01",
      "cluster": "B",
      "article_id": "S07",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "location": {
        "line": 33,
        "excerpt": "Sclar and colleagues found substantial sensitivity"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "sclar",
          "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
          "url": "https://arxiv.org/abs/2310.11324",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Author/title/year and narrowly stated finding agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S07-02",
      "cluster": "B",
      "article_id": "S07",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "location": {
        "line": 51,
        "excerpt": "reasoning tokens occupy context and are billed as output tokens"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Context, billing and hidden text distinction confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S07-03",
      "cluster": "B",
      "article_id": "S07",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "location": {
        "line": 75,
        "excerpt": "provides a shared framework for evaluating language models"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "harness",
          "title": "Language Model Evaluation Harness",
          "url": "https://github.com/EleutherAI/lm-evaluation-harness",
          "inspected_location": "README description and basic task/model arguments"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Maintainer README supports framework description.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-01",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 21,
        "excerpt": "streaming as a way to reduce waiting for visible output"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "latency",
          "title": "Latency optimization",
          "url": "https://developers.openai.com/api/docs/guides/latency-optimization",
          "inspected_location": "Generate fewer tokens; Use fewer input tokens; Make users wait less"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Streaming guidance supports the narrow statement.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-02",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 41,
        "excerpt": "separate categories and units"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Input/output/cache/tool categories confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-03",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 51,
        "excerpt": "requires matching rendered prefixes"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "cache",
          "title": "Prompt caching",
          "url": "https://developers.openai.com/api/docs/guides/prompt-caching",
          "inspected_location": "Rendered prefix; How caching works; Gotchas"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Prefix, eligibility and lookup conditions confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-04",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 43,
        "excerpt": "adding a reasoning subtotal again would overstate usage"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Reasoning is within output usage, not additive again.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-05",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 59,
        "excerpt": "Its token cost is $0.004 plus $0.004"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Invented rates: 2000×2/1e6+500×8/1e6=.008.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S08-06",
      "cluster": "B",
      "article_id": "S08",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "location": {
        "line": 61,
        "excerpt": "$0.015 and approximately $0.0167"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "1.20/80=.015; 1.50/90=.016667; examples labelled.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S09-01",
      "cluster": "B",
      "article_id": "S09",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "location": {
        "line": 43,
        "excerpt": "annotated final database state"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Database-state, policy and simulated-user description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S09-02",
      "cluster": "B",
      "article_id": "S09",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "location": {
        "line": 73,
        "excerpt": "all k independent trials succeed, averaged across tasks"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Metric definition agrees; distinct from pass@k.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S09-03",
      "cluster": "B",
      "article_id": "S09",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "location": {
        "line": 63,
        "excerpt": "reconcile state before retrying a potentially duplicated action"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "inspect",
          "title": "Inspect",
          "url": "https://inspect.aisi.org.uk/",
          "inspected_location": "Hello Inspect; task and sandbox examples"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Sound conditional recovery rule; depends on tool contract.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S09-04",
      "cluster": "B",
      "article_id": "S09",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "location": {
        "line": 75,
        "excerpt": "Do not turn a mean success rate into a reliability metric"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Average task powers differs from power of the average.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S10-01",
      "cluster": "B",
      "article_id": "S10",
      "article_file": "B/articles/maintain-llm-evaluation-suite.md",
      "location": {
        "line": 23,
        "excerpt": "monitoring for new cases and expanding the evaluation set"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Continuous evaluation and growth guidance confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S10-02",
      "cluster": "B",
      "article_id": "S10",
      "article_file": "B/articles/maintain-llm-evaluation-suite.md",
      "location": {
        "line": 51,
        "excerpt": "If the same output receives different grades"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "graders",
          "title": "Graders",
          "url": "https://developers.openai.com/api/docs/guides/graders",
          "inspected_location": "Grader types; Model graders; Grader hacking"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Scorer validation supports separating grader drift.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "B-S10-03",
      "cluster": "B",
      "article_id": "S10",
      "article_file": "B/articles/maintain-llm-evaluation-suite.md",
      "location": {
        "line": 63,
        "excerpt": "Incidents are deliberately failure-heavy"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-22",
      "exact_support_or_contradiction": "Failure-selected samples do not estimate ordinary prevalence.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-01",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 22,
        "excerpt": "MMLU, for example, is 57 tasks"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "mmlu",
          "title": "Measuring Massive Multitask Language Understanding",
          "url": "https://arxiv.org/abs/2009.03300",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "57 tasks and 2021 venue association agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-02",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 22,
        "excerpt": "204 tasks from 450 authors across 132 institutions"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "big",
          "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
          "url": "https://arxiv.org/abs/2206.04615",
          "inspected_location": "Abstract v3; bibliographic record"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "All three historical counts agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-03",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 22,
        "excerpt": "seven metrics across 16 core scenarios and 42 scenarios"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "helm",
          "title": "Holistic Evaluation of Language Models",
          "url": "https://arxiv.org/abs/2211.09110",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical metric/scenario totals agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-04",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 26,
        "excerpt": "more than 240,000 votes"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "arena",
          "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
          "url": "https://arxiv.org/abs/2403.04132",
          "inspected_location": "Abstract and author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical vote count and expert-agreement statement agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-05",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 26,
        "excerpt": "private testing, selective disclosure of results"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "illusion",
          "title": "The Leaderboard Illusion",
          "url": "https://arxiv.org/abs/2504.20879",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Study conclusions fairly attributed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-06",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 26,
        "excerpt": "24 benchmarks against 46 lifecycle best practices"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "better",
          "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
          "url": "https://arxiv.org/abs/2411.12990",
          "inspected_location": "Abstract; NeurIPS acceptance note"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Counts and reporting-gap claim agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-07",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 36,
        "excerpt": "2,294 real GitHub issues from 12 Python repositories"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swe",
          "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
          "url": "https://arxiv.org/abs/2310.06770",
          "inspected_location": "Abstract v3; task/evaluation definition"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Original dataset count agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-08",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 36,
        "excerpt": "a 500-instance Verified set"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swesite",
          "title": "SWE-bench Leaderboards",
          "url": "https://www.swebench.com/",
          "inspected_location": "Variant descriptions, not rank rows"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Human-filtered subset and Bash Only common scaffold confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-09",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 34,
        "excerpt": "Each architecture introduces a new source of nondeterminism"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "The workflow section explicitly says workflows add no new elements, only multiple model interactions. The single-agent and handoff distinctions are supported.",
      "verdict": "partially_supported",
      "severity": "minor",
      "repair": "Say architectures expose different evaluation points; workflows compound existing variation.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-10",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 36,
        "excerpt": "humans answered 92%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "gaia",
          "title": "GAIA: a benchmark for General AI Assistants",
          "url": "https://arxiv.org/abs/2311.12983",
          "inspected_location": "Abstract; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical 92% versus 15% result agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-11",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 36,
        "excerpt": "pass^8 was below 25%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical task success and retail reliability results agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-12",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 64,
        "excerpt": "8–18% HumanEval overlap"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "yang",
          "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
          "url": "https://arxiv.org/abs/2311.04850",
          "inspected_location": "Abstract v2; contamination construction and decontamination method"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Overlap finding agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-13",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 90,
        "excerpt": "roughly 900 tasks, not 120"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Independent calculation with p=.78/.74, rho=.5 gives about 894; stated rounding reasonable.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-00-14",
      "cluster": "C",
      "article_id": "ai-eval-00",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "location": {
        "line": 46,
        "excerpt": "A convenience sample of 20 examples is a demo, not an evaluation"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "card",
          "title": "With Little Power Comes Great Responsibility",
          "url": "https://arxiv.org/abs/2010.06595",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Convenience sampling weakens generalisation, but sample size alone does not define an evaluation.",
      "verdict": "unsupported_guidance",
      "severity": "substantive",
      "repair": "Describe it as a limited diagnostic evaluation; tie precision to decision.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-01",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 22,
        "excerpt": "MMLU contains 57 tasks"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "mmlu",
          "title": "Measuring Massive Multitask Language Understanding",
          "url": "https://arxiv.org/abs/2009.03300",
          "inspected_location": "Abstract; author list; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Scope and historical expert gap agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-02",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 22,
        "excerpt": "204 tasks from 450 authors across 132 institutions"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "big",
          "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
          "url": "https://arxiv.org/abs/2206.04615",
          "inspected_location": "Abstract v3; bibliographic record"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Counts and limited historical scaling finding agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-03",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 24,
        "excerpt": "17.9% of core scenarios on average"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "helm",
          "title": "Holistic Evaluation of Language Models",
          "url": "https://arxiv.org/abs/2211.09110",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "17.9→96.0% historical coverage and model/scenario counts agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-04",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 38,
        "excerpt": "more than 240,000 votes"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "arena",
          "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
          "url": "https://arxiv.org/abs/2403.04132",
          "inspected_location": "Abstract and author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical scope and crowd/expert agreement agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-05",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 40,
        "excerpt": "19.2% and 20.4%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "illusion",
          "title": "The Leaderboard Illusion",
          "url": "https://arxiv.org/abs/2504.20879",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Provider shares and 83-model 29.7% aggregate agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-01-06",
      "cluster": "C",
      "article_id": "ai-eval-01",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "location": {
        "line": 42,
        "excerpt": "24 benchmarks against 46 lifecycle best practices"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "better",
          "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
          "url": "https://arxiv.org/abs/2411.12990",
          "inspected_location": "Abstract; NeurIPS acceptance note"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Counts and reporting gaps agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-02-01",
      "cluster": "C",
      "article_id": "ai-eval-02",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "location": {
        "line": 26,
        "excerpt": "Include typical, edge and adversarial cases"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Dataset-case guidance agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-02-02",
      "cluster": "C",
      "article_id": "ai-eval-02",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "location": {
        "line": 102,
        "excerpt": "Criteria can also drift as reviewers see outputs"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "shankar",
          "title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
          "url": "https://arxiv.org/abs/2404.12272",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Criteria-drift description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-02-03",
      "cluster": "C",
      "article_id": "ai-eval-02",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "location": {
        "line": 111,
        "excerpt": "300–500 tasks:"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "card",
          "title": "With Little Power Comes Great Responsibility",
          "url": "https://arxiv.org/abs/2010.06595",
          "inspected_location": "Abstract"
        },
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Neither paper establishes this generic decision-grade range; variance, pairing, effect and power must be specified.",
      "verdict": "unsupported_guidance",
      "severity": "substantive",
      "repair": "Replace ranges with a conditional worked power calculation.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-02-04",
      "cluster": "C",
      "article_id": "ai-eval-02",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "location": {
        "line": 136,
        "excerpt": "do not generate test data with the same model"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Documented dataset sources include synthetic data; blanket same-model prohibition exceeds evidence. Risk can be controlled by independent validation.",
      "verdict": "unsupported_guidance",
      "severity": "substantive",
      "repair": "Treat generator overlap as a risk to audit; require independent labels.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-03-01",
      "cluster": "C",
      "article_id": "ai-eval-03",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "location": {
        "line": 36,
        "excerpt": "8–18% HumanEval overlap"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "yang",
          "title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
          "url": "https://arxiv.org/abs/2311.04850",
          "inspected_location": "Abstract v2; contamination construction and decontamination method"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Overlap, paraphrase-filter evasion and 13B overfit finding agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-03-02",
      "cluster": "C",
      "article_id": "ai-eval-03",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "location": {
        "line": 38,
        "excerpt": "52% and 57% respectively"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "deng",
          "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
          "url": "https://arxiv.org/abs/2311.09783",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "MMLU missing-option results and method agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-03-03",
      "cluster": "C",
      "article_id": "ai-eval-03",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "location": {
        "line": 40,
        "excerpt": "white-box, grey-box and black-box methods"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "cheng",
          "title": "A Survey on Data Contamination for Large Language Models",
          "url": "https://arxiv.org/abs/2502.14425",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Survey taxonomy agrees; source is synthesis, not an original experiment.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-03-04",
      "cluster": "C",
      "article_id": "ai-eval-03",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "location": {
        "line": 44,
        "excerpt": "initial release contained 400 problems published between May 2023 and May 2024"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "live",
          "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
          "url": "https://arxiv.org/abs/2403.07974",
          "inspected_location": "Abstracts v1 and v2; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "400 is corroborated; May 2024 window is v2. v1 ends February 2024.",
      "verdict": "inaccurate_version_label",
      "severity": "minor",
      "repair": "Call this the June 2024 paper snapshot, not the initial release.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-03-05",
      "cluster": "C",
      "article_id": "ai-eval-03",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "location": {
        "line": 88,
        "excerpt": "treat the score as an upper bound on likely performance"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "oren",
          "title": "Proving Test Set Contamination in Black Box Language Models",
          "url": "https://arxiv.org/abs/2310.17623",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Unclear provider filtering supplies no mathematical upper bound on deployment performance.",
      "verdict": "incorrect_guidance",
      "severity": "substantive",
      "repair": "State training exposure is unknown; test fresh workload cases without claiming a bound.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-04-01",
      "cluster": "C",
      "article_id": "ai-eval-04",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "location": {
        "line": 30,
        "excerpt": "1 - C(n-c, k) / C(n, k)"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "code",
          "title": "Evaluating Large Language Models Trained on Code",
          "url": "https://arxiv.org/abs/2107.03374",
          "inspected_location": "Abstract; HTML v2 section 2.1 and Appendix A"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Correct per-task estimator; specify n≥k and task averaging.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-04-02",
      "cluster": "C",
      "article_id": "ai-eval-04",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "location": {
        "line": 32,
        "excerpt": "Measures whether all k attempts succeed"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Correct reliability-event definition; not a guarantee of working every time.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-04-03",
      "cluster": "C",
      "article_id": "ai-eval-04",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "location": {
        "line": 34,
        "excerpt": "SWE-bench scores a model by whether its patch resolves the issue"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swe",
          "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
          "url": "https://arxiv.org/abs/2310.06770",
          "inspected_location": "Abstract v3; task/evaluation definition"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Test-based resolution description agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-04-04",
      "cluster": "C",
      "article_id": "ai-eval-04",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "location": {
        "line": 74,
        "excerpt": "Macro-averaging (average of stratum scores)"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Macro intuition sound; micro pooled counts equals traffic evidence only with representative sampling.",
      "verdict": "needs_clarification",
      "severity": "minor",
      "repair": "Qualify traffic interpretation by prevalence and sampling.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-04-05",
      "cluster": "C",
      "article_id": "ai-eval-04",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "location": {
        "line": 92,
        "excerpt": "Longer answers often score higher with model judges and with token-overlap metrics"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "metrics",
          "title": "precision_recall_fscore_support",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
          "inspected_location": "Definitions and average parameter"
        },
        {
          "source_id": "judge",
          "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
          "url": "https://arxiv.org/abs/2306.05685",
          "inspected_location": "Abstract v4; HTML sections 3.3–3.4"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Judge length bias is documented. F1 penalises unmatched tokens: reference a, prediction a scores 1; prediction a b scores 2/3. Citation does not establish the broad overlap-metric claim.",
      "verdict": "partially_supported",
      "severity": "substantive",
      "repair": "Separate LLM length bias from metric-specific overlap incentives.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-05-01",
      "cluster": "C",
      "article_id": "ai-eval-05",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "location": {
        "line": 72,
        "excerpt": "over 80% of the time"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "judge",
          "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
          "url": "https://arxiv.org/abs/2306.05685",
          "inspected_location": "Abstract v4; HTML sections 3.3–3.4"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical study agreement accurately qualified by its setting.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-05-02",
      "cluster": "C",
      "article_id": "ai-eval-05",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "location": {
        "line": 38,
        "excerpt": "criteria drift"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "shankar",
          "title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
          "url": "https://arxiv.org/abs/2404.12272",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Definition and iterative-validation lesson agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-05-03",
      "cluster": "C",
      "article_id": "ai-eval-05",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "location": {
        "line": 82,
        "excerpt": "the average cancels the position effect"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "position",
          "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
          "url": "https://arxiv.org/abs/2406.07791",
          "inspected_location": "Abstract; author list; submission history"
        },
        {
          "source_id": "judge",
          "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
          "url": "https://arxiv.org/abs/2306.05685",
          "inspected_location": "Abstract v4; HTML sections 3.3–3.4"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Averaging neutralises the stated pure first-position preference. It does not establish removal of general task/candidate-dependent bias; encoding categorical verdicts and retaining disagreements are unspecified.",
      "verdict": "needs_clarification",
      "severity": "substantive",
      "repair": "Define vote aggregation, retain reversals and tie/adjudication rules; limit cancellation to the symmetric case.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-05-04",
      "cluster": "C",
      "article_id": "ai-eval-05",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "location": {
        "line": 84,
        "excerpt": "Self-enhancement bias is real"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "judge",
          "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
          "url": "https://arxiv.org/abs/2306.05685",
          "inspected_location": "Abstract v4; HTML sections 3.3–3.4"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Original section 3.3 explicitly cannot establish self-enhancement from its limited data.",
      "verdict": "overstated_evidence",
      "severity": "substantive",
      "repair": "Describe a potential mechanism and the study’s uncertainty.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-05-05",
      "cluster": "C",
      "article_id": "ai-eval-05",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "location": {
        "line": 60,
        "excerpt": "kappa = (p_o - p_e) / (1 - p_e)"
      },
      "claim_type": "method_definition",
      "sources": [
        {
          "source_id": "kappa",
          "title": "cohen_kappa_score",
          "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.cohen_kappa_score.html",
          "inspected_location": "Definition; weights parameter"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Formula matches official documentation; draft lacks a nearby agreement-statistics source. Alpha preference is not justified over weighted kappa.",
      "verdict": "supported_with_citation_gap",
      "severity": "minor",
      "repair": "Add an authoritative reliability-statistics reference; explain weighted kappa as an ordinal option.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-01",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 28,
        "excerpt": "Madaan et al., 2024"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "madaan",
          "title": "Quantifying Variance in Evaluation Benchmarks",
          "url": "https://arxiv.org/abs/2406.10229",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Bibliography and studied variance/format findings agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-02",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 34,
        "excerpt": "SE = sd / sqrt(n)"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Independent-unit SE formula agrees; normal approximation needs boundary/sample caveats.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-03",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 50,
        "excerpt": "correlation of about 0.5"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Independent arithmetic: paired SE .03898, interval −.0364 to .1164; roughly 894 tasks for .04 effect.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-04",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 58,
        "excerpt": "up to about three times larger"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Table 4 reaches ratio 3.05.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-05",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 66,
        "excerpt": "two samples reduce total variance by about a third"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Matches the uniform-task-difficulty binary example, not a universal rate.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-06",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 76,
        "excerpt": "pass^8 below 25%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical result agrees; pass@k versus pass^k distinction sound.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-07",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 82,
        "excerpt": "2,000 sentences had about 75% power"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "card",
          "title": "With Little Power Comes Great Responsibility",
          "url": "https://arxiv.org/abs/2010.06595",
          "inspected_location": "Abstract"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical MT power result agrees with its assumptions.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-08",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 88,
        "excerpt": "about 969 independent questions"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Paper’s stated-assumption power example agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-06-09",
      "cluster": "C",
      "article_id": "ai-eval-06",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "location": {
        "line": 58,
        "excerpt": "Ignoring clustering understates the standard error"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "groups",
          "title": "Cross-validation: evaluating estimator performance",
          "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
          "inspected_location": "Grouped and time-series data sections"
        },
        {
          "source_id": "miller",
          "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
          "url": "https://arxiv.org/abs/2411.00640",
          "inspected_location": "Full HTML v1 sections 2–5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Positive dependence often does; negative/zero within-cluster correlation need not.",
      "verdict": "needs_clarification",
      "severity": "minor",
      "repair": "Use can understate; explain that 50 documents is not automatically effective n=50.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-07-01",
      "cluster": "C",
      "article_id": "ai-eval-07",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "location": {
        "line": 14,
        "excerpt": "single call to a multi-agent handoff"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Architecture-related nondeterminism guidance agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-07-02",
      "cluster": "C",
      "article_id": "ai-eval-07",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "location": {
        "line": 74,
        "excerpt": "Inspect makes datasets, solvers, scorers and sandboxes explicit"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "inspect",
          "title": "Inspect",
          "url": "https://inspect.aisi.org.uk/",
          "inspected_location": "Hello Inspect; task and sandbox examples"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Components and sandbox support confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-07-03",
      "cluster": "C",
      "article_id": "ai-eval-07",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "location": {
        "line": 100,
        "excerpt": "same mini-SWE-agent environment"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swesite",
          "title": "SWE-bench Leaderboards",
          "url": "https://www.swebench.com/",
          "inspected_location": "Variant descriptions, not rank rows"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Bash Only scaffold description confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-07-04",
      "cluster": "C",
      "article_id": "ai-eval-07",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "location": {
        "line": 110,
        "excerpt": "always increases tokens and latency"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Adaptive, model-dependent effort is not a per-request monotonic guarantee.",
      "verdict": "overstated_evidence",
      "severity": "substantive",
      "repair": "Say it usually trades increased work for quality; measure actual usage and latency.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-07-05",
      "cluster": "C",
      "article_id": "ai-eval-07",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "location": {
        "line": 114,
        "excerpt": "the same nominal setting across models"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        },
        {
          "source_id": "sclar",
          "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
          "url": "https://arxiv.org/abs/2310.11324",
          "inspected_location": "Abstract v2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "The word nominal is appropriate, but calling this matched effort risks implying equivalent work. Settings are model-dependent; label equality alone establishes no compute equivalence.",
      "verdict": "needs_clarification",
      "severity": "substantive",
      "repair": "Call it matched setting labels, disclose unsupported controls and compare measured cost/latency envelopes.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-01",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 36,
        "excerpt": "cutting output tokens by 50% may cut latency by roughly 50%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "latency",
          "title": "Latency optimization",
          "url": "https://developers.openai.com/api/docs/guides/latency-optimization",
          "inspected_location": "Generate fewer tokens; Use fewer input tokens; Make users wait less"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Both output and 1–5% input heuristics accurately qualified.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-02",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 47,
        "excerpt": "at least 25,000 tokens"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Initial-experiment buffer recommendation confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-03",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 44,
        "excerpt": "Output tokens:"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "reason",
          "title": "Reasoning models",
          "url": "https://developers.openai.com/api/docs/guides/reasoning",
          "inspected_location": "Reasoning effort; How reasoning works; Allocating space for reasoning"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Article defines output as visible text, then uses output alone in billing formula. Billed output includes reasoning, so following the visible-only definition undercounts.",
      "verdict": "incorrect_guidance",
      "severity": "publication_blocker",
      "repair": "Define billed output including reasoning/formatting, or sum disjoint categories once.",
      "publication_blocker": true
    },
    {
      "claim_id": "C-ai-eval-08-04",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 72,
        "excerpt": "GPT-6 Astra | $10.00 | $50.00"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Standard/cache/batch values agree with live official tables.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-05",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 73,
        "excerpt": "GPT-6 Luna | $0.10 | $0.50"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Standard/cache/batch values agree with live official tables.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-06",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 74,
        "excerpt": "Claude Fable 5.1 | $10.00 | $50.00"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "claude",
          "title": "Pricing",
          "url": "https://claude.com/pricing",
          "inspected_location": "Latest models: Fable 5.1 and Haiku 5.5"
        },
        {
          "source_id": "claudedetail",
          "title": "Pricing",
          "url": "https://platform.claude.com/docs/en/about-claude/pricing",
          "inspected_location": "Batch processing; Long context pricing"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Rates and 50% batch discount confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-07",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 75,
        "excerpt": "Claude Haiku 5.5 | $0.10 | $0.50"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "claude",
          "title": "Pricing",
          "url": "https://claude.com/pricing",
          "inspected_location": "Latest models: Fable 5.1 and Haiku 5.5"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Rates apply only through 100K prompt tokens. Table does not disclose the fivefold long-context input/output uplift.",
      "verdict": "missing_material_condition",
      "severity": "substantive",
      "repair": "Label short-prompt rates and add the >100K bracket.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-08",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 76,
        "excerpt": "Gemini 4 Argon | $2.00 | $10.00"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "gemini",
          "title": "Gemini Developer API pricing",
          "url": "https://ai.google.dev/gemini-api/docs/pricing",
          "inspected_location": "Gemini 4 Argon Standard and Batch tables"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Standard/cache/batch values and dated promotion confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-09",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 78,
        "excerpt": "double on 1 December 2026"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "gemini",
          "title": "Gemini Developer API pricing",
          "url": "https://ai.google.dev/gemini-api/docs/pricing",
          "inspected_location": "Gemini 4 Argon Standard and Batch tables"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Dated doubling is explicitly on official page.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-10",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 102,
        "excerpt": "about $1.80 more for each additional success"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Arithmetic: (27.50−.275)/15=1.815; .32/.0032 and .0039 per-success examples round correctly. Hypothetical.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-08-11",
      "cluster": "C",
      "article_id": "ai-eval-08",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "location": {
        "line": 104,
        "excerpt": "roughly $0.24"
      },
      "claim_type": "illustrative_calculation",
      "sources": [
        {
          "source_id": "price",
          "title": "Pricing",
          "url": "https://developers.openai.com/api/docs/pricing",
          "inspected_location": "Flagship Standard and Flex/Batch tables; Tools"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "(.075+.0075+.10)×1.1/.85=.23618; correct for assumed cache reads, before cache-write charges.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-01",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 37,
        "excerpt": "best model of the time resolved only 1.96%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swe",
          "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
          "url": "https://arxiv.org/abs/2310.06770",
          "inspected_location": "Abstract v3; task/evaluation definition"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical original benchmark result agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-02",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 37,
        "excerpt": "300-instance Multilingual set across 42 repositories and nine languages"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "swesite",
          "title": "SWE-bench Leaderboards",
          "url": "https://www.swebench.com/",
          "inspected_location": "Variant descriptions, not rank rows"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "All subset counts and common-scaffold description confirmed.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-03",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 39,
        "excerpt": "pass^8 was below 25%"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Historical result and final-state protocol agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-04",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 41,
        "excerpt": "GAIA** contains 466"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "gaia",
          "title": "GAIA: a benchmark for General AI Assistants",
          "url": "https://arxiv.org/abs/2311.12983",
          "inspected_location": "Abstract; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "466 total/300 retained answers and 92%/15% results agree.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-05",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 43,
        "excerpt": "problems published after their training cut-off"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "live",
          "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
          "url": "https://arxiv.org/abs/2403.07974",
          "inspected_location": "Abstracts v1 and v2; submission history"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Temporal selection reduces exposure risk; not proof of training cutoff or zero contamination.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-06",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 73,
        "excerpt": "starting with traces when debugging behaviour"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "agentdocs",
          "title": "Evaluate agent workflows",
          "url": "https://developers.openai.com/api/docs/guides/agent-evals",
          "inspected_location": "Start with traces; Move to datasets and eval runs"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Traces-first/datasets-next guidance agrees.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-07",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 85,
        "excerpt": "Inspect supports composable datasets, agents, tools, scorers and sandboxes"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "inspect",
          "title": "Inspect",
          "url": "https://inspect.aisi.org.uk/",
          "inspected_location": "Hello Inspect; task and sandbox examples"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Official components and sandbox example support this.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-09-08",
      "cluster": "C",
      "article_id": "ai-eval-09",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "location": {
        "line": 79,
        "excerpt": "only 38% pass^5"
      },
      "claim_type": "illustrative_example",
      "sources": [
        {
          "source_id": "tau",
          "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
          "url": "https://arxiv.org/abs/2406.12045",
          "inspected_location": "Abstract; HTML v1 section 3"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Plausible illustrative joint reliability; neither this nor pass@1 alone predicts an unattended workload’s experience.",
      "verdict": "needs_clarification",
      "severity": "minor",
      "repair": "Report both metrics and workload assumptions; avoid claiming one universally predicts production.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-10-01",
      "cluster": "C",
      "article_id": "ai-eval-10",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "location": {
        "line": 45,
        "excerpt": "most benchmarks do not report statistical significance"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "better",
          "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
          "url": "https://arxiv.org/abs/2411.12990",
          "inspected_location": "Abstract; NeurIPS acceptance note"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Accurately reflects the surveyed benchmarks, but avoid implying every current benchmark.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-10-02",
      "cluster": "C",
      "article_id": "ai-eval-10",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "location": {
        "line": 75,
        "excerpt": "selection effects and unequal access can distort"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "illusion",
          "title": "The Leaderboard Illusion",
          "url": "https://arxiv.org/abs/2504.20879",
          "inspected_location": "Abstract v2; author list"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Study-specific transfer is reasonably qualified.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-10-03",
      "cluster": "C",
      "article_id": "ai-eval-10",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "location": {
        "line": 112,
        "excerpt": "31 October 2026"
      },
      "claim_type": "sourced_fact",
      "sources": [
        {
          "source_id": "eval",
          "title": "Evaluation best practices",
          "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
          "inspected_location": "Evals tips; Design your eval process; Evaluation architectures; Model-based evals; deprecation notice"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Live notice corroborates both scheduled read-only and shutdown dates.",
      "verdict": "supported",
      "severity": "none",
      "repair": null,
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-10-04",
      "cluster": "C",
      "article_id": "ai-eval-10",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "location": {
        "line": 108,
        "excerpt": "retire a set after it has been used for a decision"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "leak",
          "title": "Common pitfalls and recommended practices",
          "url": "https://scikit-learn.org/stable/common_pitfalls.html",
          "inspected_location": "Data leakage, section 12.2"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Exposure/tuning weakens independence; a universal one-use rule is editorial choice. Contradicts own two-decision retirement template in article 03.",
      "verdict": "inconsistent_guidance",
      "severity": "minor",
      "repair": "Choose one declared access/retirement policy and separate sealed selection from visible regression use.",
      "publication_blocker": false
    },
    {
      "claim_id": "C-ai-eval-10-05",
      "cluster": "C",
      "article_id": "ai-eval-10",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "location": {
        "line": 114,
        "excerpt": "Retire what no longer discriminates"
      },
      "claim_type": "editorial_guidance",
      "sources": [
        {
          "source_id": "checklist",
          "title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
          "url": "https://aclanthology.org/2020.acl-main.442/",
          "inspected_location": "ACL abstract and citation record"
        }
      ],
      "retrieval_date_utc": "2026-10-25",
      "submission_research_date": "2026-10-25",
      "exact_support_or_contradiction": "Saturated cases may still enforce critical regression gates; removing all such cases is unsafe as a blanket instruction.",
      "verdict": "needs_clarification",
      "severity": "minor",
      "repair": "Retain critical invariant tests; refresh selection-oriented tasks separately.",
      "publication_blocker": false
    }
  ],
  "claim_record_count": 162,
  "counts_per_article": {
    "A/articles/agent-system-evaluation.md": 4,
    "A/articles/build-task-eval.md": 3,
    "A/articles/choosing-model-benchmark.md": 5,
    "A/articles/datasets-holdouts-contamination.md": 4,
    "A/articles/fair-model-comparisons.md": 3,
    "A/articles/human-evaluation-model-judges.md": 4,
    "A/articles/latency-cost.md": 4,
    "A/articles/maintaining-evaluations.md": 3,
    "A/articles/metrics-rubrics.md": 4,
    "A/articles/public-benchmark-literacy.md": 4,
    "A/articles/repeatability-uncertainty.md": 4,
    "B/articles/ai-latency-cost-per-successful-task.md": 6,
    "B/articles/ai-model-benchmarking-guide.md": 4,
    "B/articles/benchmark-uncertainty-repeatability.md": 5,
    "B/articles/design-llm-evaluation-tasks.md": 3,
    "B/articles/evaluate-ai-agents-tools-outcomes.md": 4,
    "B/articles/evaluation-dataset-leakage-holdouts.md": 4,
    "B/articles/fair-ai-model-comparisons.md": 3,
    "B/articles/human-reviewers-llm-judges.md": 4,
    "B/articles/llm-evaluation-metrics-rubrics.md": 3,
    "B/articles/maintain-llm-evaluation-suite.md": 3,
    "B/articles/read-ai-benchmark-leaderboards.md": 4,
    "C/articles/contamination-leakage-and-holdouts.md": 5,
    "C/articles/designing-evaluation-tasks-and-datasets.md": 4,
    "C/articles/evaluating-agents-and-tool-use.md": 8,
    "C/articles/evaluating-ai-models-and-agent-systems.md": 14,
    "C/articles/human-evaluation-and-model-judges.md": 5,
    "C/articles/interpreting-results-and-maintaining-evals.md": 5,
    "C/articles/latency-token-usage-and-cost.md": 11,
    "C/articles/objective-metrics-and-rubrics.md": 5,
    "C/articles/prompt-tool-and-configuration-fairness.md": 5,
    "C/articles/public-benchmarks-vs-workload-evaluation.md": 6,
    "C/articles/repeatability-variance-and-uncertainty.md": 9
  },
  "bibliographic_checks": [
    {
      "cluster": "A",
      "submitted_source_id": "openai-eval-best-practices",
      "submitted_title": "Evaluation best practices | OpenAI API",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "long-running-work",
      "submitted_title": "Long-running work | ChatGPT Learn",
      "submitted_url": "https://learn.chatgpt.com/docs/long-running-work",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "workflow",
      "verified_title": "Long-running work",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "helm",
      "submitted_title": "Holistic Evaluation of Language Models",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "mmlu",
      "submitted_title": "Measuring Massive Multitask Language Understanding",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "rephrased-contamination",
      "submitted_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "contamination-2026",
      "submitted_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards",
      "submitted_url": "https://arxiv.org/abs/2609.02899",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "xiao",
      "verified_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards",
      "verified_authors": "Xingyao Xiao and Yihong Cheng",
      "verified_year": "2026",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "judge-position-bias",
      "submitted_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
      "submitted_url": "https://arxiv.org/abs/2406.07791",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "position",
      "verified_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
      "verified_authors": "Lin Shi et al.",
      "verified_year": "2024 (revised 2025)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "openai-graders",
      "submitted_title": "Graders | OpenAI API",
      "submitted_url": "https://developers.openai.com/api/docs/guides/graders",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "graders",
      "verified_title": "Graders",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "benchmark-variance",
      "submitted_title": "Accounting for Variance in Machine Learning Benchmarks",
      "submitted_url": "https://arxiv.org/abs/2103.03098",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "variance",
      "verified_title": "Accounting for Variance in Machine Learning Benchmarks",
      "verified_authors": "Xavier Bouthillier et al.",
      "verified_year": "2021",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "openai-pricing",
      "submitted_title": "Pricing | OpenAI API",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "openai-agent-evals",
      "submitted_title": "Evaluate agent workflows | OpenAI API",
      "submitted_url": "https://developers.openai.com/api/docs/guides/agent-evals",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "agentdocs",
      "verified_title": "Evaluate agent workflows",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "swebench",
      "submitted_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "submitted_source_id": "swebench-pro-verified",
      "submitted_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents",
      "submitted_url": "https://arxiv.org/abs/2609.08149",
      "submitted_author_label": null,
      "submitted_date_label": null,
      "verified_source_id": "zhengpro",
      "verified_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents",
      "verified_authors": "Pujun Zheng et al.",
      "verified_year": "2026",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R01",
      "submitted_title": "Evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R02",
      "submitted_title": "Long-running work",
      "submitted_url": "https://learn.chatgpt.com/docs/long-running-work",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "workflow",
      "verified_title": "Long-running work",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R03",
      "submitted_title": "Measuring Massive Multitask Language Understanding",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "submitted_author_label": "Hendrycks et al.",
      "submitted_date_label": null,
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R04",
      "submitted_title": "Holistic Evaluation of Language Models",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "submitted_author_label": "Liang et al.",
      "submitted_date_label": null,
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R05",
      "submitted_title": "Language Model Evaluation Harness",
      "submitted_url": "https://github.com/EleutherAI/lm-evaluation-harness",
      "submitted_author_label": "EleutherAI",
      "submitted_date_label": null,
      "verified_source_id": "harness",
      "verified_title": "Language Model Evaluation Harness",
      "verified_authors": "EleutherAI",
      "verified_year": "living repository",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R06",
      "submitted_title": "SWE-bench Leaderboards",
      "submitted_url": "https://www.swebench.com/",
      "submitted_author_label": "SWE-bench maintainers",
      "submitted_date_label": null,
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R07",
      "submitted_title": "Evaluating Large Language Models Trained on Code",
      "submitted_url": "https://arxiv.org/abs/2107.03374",
      "submitted_author_label": "Chen et al.",
      "submitted_date_label": null,
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R08",
      "submitted_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
      "submitted_url": "https://aclanthology.org/2020.acl-main.442/",
      "submitted_author_label": "Ribeiro et al.",
      "submitted_date_label": null,
      "verified_source_id": "checklist",
      "verified_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
      "verified_authors": "Marco Tulio Ribeiro et al.",
      "verified_year": "2020",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R09",
      "submitted_title": "Common pitfalls and recommended practices",
      "submitted_url": "https://scikit-learn.org/stable/common_pitfalls.html",
      "submitted_author_label": "scikit-learn developers",
      "submitted_date_label": null,
      "verified_source_id": "leak",
      "verified_title": "Common pitfalls and recommended practices",
      "verified_authors": "scikit-learn developers",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R10",
      "submitted_title": "Detecting Pretraining Data from Large Language Models",
      "submitted_url": "https://arxiv.org/abs/2310.16789",
      "submitted_author_label": "Shi et al.",
      "submitted_date_label": null,
      "verified_source_id": "shi",
      "verified_title": "Detecting Pretraining Data from Large Language Models",
      "verified_authors": "Weijia Shi et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R11",
      "submitted_title": "Datasheets for Datasets",
      "submitted_url": "https://arxiv.org/abs/1803.09010",
      "submitted_author_label": "Gebru et al.",
      "submitted_date_label": null,
      "verified_source_id": "datasheet",
      "verified_title": "Datasheets for Datasets",
      "verified_authors": "Timnit Gebru et al.",
      "verified_year": "2018 (CACM 2021)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R12",
      "submitted_title": "Proving Test Set Contamination in Black Box Language Models",
      "submitted_url": "https://arxiv.org/abs/2310.17623",
      "submitted_author_label": "Oren et al.",
      "submitted_date_label": null,
      "verified_source_id": "oren",
      "verified_title": "Proving Test Set Contamination in Black Box Language Models",
      "verified_authors": "Yonatan Oren et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R13",
      "submitted_title": "precision_recall_fscore_support",
      "submitted_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
      "submitted_author_label": "scikit-learn developers",
      "submitted_date_label": null,
      "verified_source_id": "metrics",
      "verified_title": "precision_recall_fscore_support",
      "verified_authors": "scikit-learn developers",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R14",
      "submitted_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "submitted_url": "https://arxiv.org/abs/2306.05685",
      "submitted_author_label": "Zheng et al.",
      "submitted_date_label": null,
      "verified_source_id": "judge",
      "verified_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "verified_authors": "Lianmin Zheng et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R15",
      "submitted_title": "7.2.4.1. Confidence intervals",
      "submitted_url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm",
      "submitted_author_label": "NIST/SEMATECH e-Handbook",
      "submitted_date_label": null,
      "verified_source_id": "wilson",
      "verified_title": "7.2.4.1. Confidence intervals",
      "verified_authors": "NIST/SEMATECH",
      "verified_year": "living reference",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R16",
      "submitted_title": "scipy.stats.bootstrap",
      "submitted_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
      "submitted_author_label": "SciPy developers",
      "submitted_date_label": null,
      "verified_source_id": "bootstrap",
      "verified_title": "scipy.stats.bootstrap",
      "verified_authors": "SciPy developers",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R17",
      "submitted_title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
      "submitted_url": "https://arxiv.org/abs/2310.11324",
      "submitted_author_label": "Sclar et al.",
      "submitted_date_label": null,
      "verified_source_id": "sclar",
      "verified_title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
      "verified_authors": "Melanie Sclar et al.",
      "verified_year": "2023 (ICLR 2024)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R18",
      "submitted_title": "Reasoning models",
      "submitted_url": "https://developers.openai.com/api/docs/guides/reasoning",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "reason",
      "verified_title": "Reasoning models",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R19",
      "submitted_title": "Latency optimization",
      "submitted_url": "https://developers.openai.com/api/docs/guides/latency-optimization",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "latency",
      "verified_title": "Latency optimization",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R20",
      "submitted_title": "Pricing",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R21",
      "submitted_title": "Prompt caching",
      "submitted_url": "https://developers.openai.com/api/docs/guides/prompt-caching",
      "submitted_author_label": "OpenAI",
      "submitted_date_label": null,
      "verified_source_id": "cache",
      "verified_title": "Prompt caching",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "B",
      "submitted_source_id": "R22",
      "submitted_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "submitted_author_label": "Yao et al.",
      "submitted_date_label": null,
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-openai-eval",
      "submitted_title": "Evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "submitted_author_label": [
        "OpenAI"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-openai-agent-evals",
      "submitted_title": "Evaluate agent workflows",
      "submitted_url": "https://developers.openai.com/api/docs/guides/agent-evals",
      "submitted_author_label": [
        "OpenAI"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "agentdocs",
      "verified_title": "Evaluate agent workflows",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-openai-pricing",
      "submitted_title": "Pricing",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "submitted_author_label": [
        "OpenAI"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-openai-latency",
      "submitted_title": "Latency optimization",
      "submitted_url": "https://developers.openai.com/api/docs/guides/latency-optimization",
      "submitted_author_label": [
        "OpenAI"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "latency",
      "verified_title": "Latency optimization",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-openai-reasoning",
      "submitted_title": "Reasoning models",
      "submitted_url": "https://developers.openai.com/api/docs/guides/reasoning",
      "submitted_author_label": [
        "OpenAI"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "reason",
      "verified_title": "Reasoning models",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-anthropic-pricing",
      "submitted_title": "Plans and pricing",
      "submitted_url": "https://claude.com/pricing",
      "submitted_author_label": [
        "Anthropic"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "claude",
      "verified_title": "Pricing",
      "verified_authors": "Anthropic",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-google-pricing",
      "submitted_title": "Gemini Developer API pricing",
      "submitted_url": "https://ai.google.dev/gemini-api/docs/pricing",
      "submitted_author_label": [
        "Google"
      ],
      "submitted_date_label": "2026-10-22",
      "verified_source_id": "gemini",
      "verified_title": "Gemini Developer API pricing",
      "verified_authors": "Google",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-helm",
      "submitted_title": "Holistic Evaluation of Language Models",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "submitted_author_label": [
        "Liang, P.",
        "Bommasani, R.",
        "Lee, T.",
        "Tsipras, D.",
        "Soylu, D.",
        "Yasunaga, M.",
        "Zhang, Y.",
        "Narayanan, D.",
        "Wu, Y.",
        "Kumar, A.",
        "and others"
      ],
      "submitted_date_label": "2022-11-16",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-mmlu",
      "submitted_title": "Measuring Massive Multitask Language Understanding",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "submitted_author_label": [
        "Hendrycks, D.",
        "Burns, C.",
        "Basart, S.",
        "Zou, A.",
        "Mazeika, M.",
        "Song, D.",
        "Steinhardt, J."
      ],
      "submitted_date_label": "2020-09-07",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-bigbench",
      "submitted_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "submitted_url": "https://arxiv.org/abs/2206.04615",
      "submitted_author_label": [
        "Srivastava, A.",
        "Rastogi, A.",
        "Rao, A.",
        "and others"
      ],
      "submitted_date_label": "2022-06-09",
      "verified_source_id": "big",
      "verified_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "verified_authors": "Aarohi Srivastava et al.",
      "verified_year": "2022",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-arena",
      "submitted_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
      "submitted_url": "https://arxiv.org/abs/2403.04132",
      "submitted_author_label": [
        "Chiang, W.-L.",
        "Zheng, L.",
        "Sheng, Y.",
        "Angelopoulos, A. N.",
        "Li, T.",
        "Li, D.",
        "Zhang, H.",
        "Zhu, B.",
        "Jordan, M.",
        "Gonzalez, J. E.",
        "Stoica, I."
      ],
      "submitted_date_label": "2024-03-07",
      "verified_source_id": "arena",
      "verified_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
      "verified_authors": "Wei-Lin Chiang et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-mtbench",
      "submitted_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "submitted_url": "https://arxiv.org/abs/2306.05685",
      "submitted_author_label": [
        "Zheng, L.",
        "Chiang, W.-L.",
        "Sheng, Y.",
        "Zhuang, S.",
        "Wu, Z.",
        "Zhuang, Y.",
        "Lin, Z.",
        "Li, Z.",
        "Li, D.",
        "Xing, E. P.",
        "Zhang, H.",
        "Gonzalez, J. E.",
        "Stoica, I."
      ],
      "submitted_date_label": "2023-06-09",
      "verified_source_id": "judge",
      "verified_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "verified_authors": "Lianmin Zheng et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-miller",
      "submitted_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "submitted_url": "https://arxiv.org/abs/2411.00640",
      "submitted_author_label": [
        "Miller, E."
      ],
      "submitted_date_label": "2024-11-01",
      "verified_source_id": "miller",
      "verified_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "verified_authors": "Evan Miller",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-kapoor",
      "submitted_title": "AI Agents That Matter",
      "submitted_url": "https://arxiv.org/abs/2407.01502",
      "submitted_author_label": [
        "Kapoor, S.",
        "Stroebl, B.",
        "Siegel, Z. S.",
        "Nadgir, N.",
        "Narayanan, A."
      ],
      "submitted_date_label": "2024-07-01",
      "verified_source_id": "kapoor",
      "verified_title": "AI Agents That Matter",
      "verified_authors": "Sayash Kapoor et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-swebench-paper",
      "submitted_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "submitted_author_label": [
        "Jimenez, C. E.",
        "Yang, J.",
        "Wettig, A.",
        "Yao, S.",
        "Pei, K.",
        "Press, O.",
        "Narasimhan, K."
      ],
      "submitted_date_label": "2023-10-10",
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-swebench-site",
      "submitted_title": "SWE-bench Leaderboards",
      "submitted_url": "https://www.swebench.com/",
      "submitted_author_label": [
        "SWE-bench team"
      ],
      "submitted_date_label": "2026",
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-livecodebench",
      "submitted_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
      "submitted_url": "https://arxiv.org/abs/2403.07974",
      "submitted_author_label": [
        "Jain, N.",
        "Han, K.",
        "Gu, A.",
        "Li, W.-D.",
        "Yan, F.",
        "Zhang, T.",
        "Wang, S.",
        "Solar-Lezama, A.",
        "Sen, K.",
        "Stoica, I."
      ],
      "submitted_date_label": "2024-03-12",
      "verified_source_id": "live",
      "verified_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
      "verified_authors": "Naman Jain et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-taubench",
      "submitted_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "submitted_author_label": [
        "Yao, S.",
        "Shinn, N.",
        "Razavi, P.",
        "Narasimhan, K."
      ],
      "submitted_date_label": "2024-06-17",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-gaia",
      "submitted_title": "GAIA: a benchmark for General AI Assistants",
      "submitted_url": "https://arxiv.org/abs/2311.12983",
      "submitted_author_label": [
        "Mialon, G.",
        "Fourrier, C.",
        "Swift, C.",
        "Wolf, T.",
        "LeCun, Y.",
        "Scialom, T."
      ],
      "submitted_date_label": "2023-11-21",
      "verified_source_id": "gaia",
      "verified_title": "GAIA: a benchmark for General AI Assistants",
      "verified_authors": "Grégoire Mialon et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-betterbench",
      "submitted_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "submitted_url": "https://arxiv.org/abs/2411.12990",
      "submitted_author_label": [
        "Reuel, A.",
        "Hardy, A.",
        "Smith, C.",
        "Lamparth, M.",
        "Hardy, M.",
        "Kochenderfer, M. J."
      ],
      "submitted_date_label": "2024-11-20",
      "verified_source_id": "better",
      "verified_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "verified_authors": "Anka Reuel et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-leaderboard-illusion",
      "submitted_title": "The Leaderboard Illusion",
      "submitted_url": "https://arxiv.org/abs/2504.20879",
      "submitted_author_label": [
        "Singh, S.",
        "Nan, Y.",
        "Wang, A.",
        "D'Souza, D.",
        "Kapoor, S.",
        "Üstün, A.",
        "Koyejo, S.",
        "Deng, Y.",
        "Longpre, S.",
        "Smith, N. A.",
        "Ermis, B.",
        "Fadaee, M.",
        "Hooker, S."
      ],
      "submitted_date_label": "2025-04-29",
      "verified_source_id": "illusion",
      "verified_title": "The Leaderboard Illusion",
      "verified_authors": "Shivalika Singh et al.",
      "verified_year": "2025",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-deng-contamination",
      "submitted_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
      "submitted_url": "https://arxiv.org/abs/2311.09783",
      "submitted_author_label": [
        "Deng, C.",
        "Zhao, Y.",
        "Tang, X.",
        "Gerstein, M.",
        "Cohan, A."
      ],
      "submitted_date_label": "2023-11-16",
      "verified_source_id": "deng",
      "verified_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
      "verified_authors": "Chunyuan Deng et al.",
      "verified_year": "2023 (NAACL 2024)",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-yang-contamination",
      "submitted_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "submitted_author_label": [
        "Yang, S.",
        "Chiang, W.-L.",
        "Zheng, L.",
        "Gonzalez, J. E.",
        "Stoica, I."
      ],
      "submitted_date_label": "2023-11-08",
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-cheng-contamination",
      "submitted_title": "A Survey on Data Contamination for Large Language Models",
      "submitted_url": "https://arxiv.org/abs/2502.14425",
      "submitted_author_label": [
        "Cheng, Y.",
        "Chang, Y.",
        "Wu, Y."
      ],
      "submitted_date_label": "2025-02-20",
      "verified_source_id": "cheng",
      "verified_title": "A Survey on Data Contamination for Large Language Models",
      "verified_authors": "Yuxing Cheng, Yi Chang and Yuan Wu",
      "verified_year": "2025",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-codex-paper",
      "submitted_title": "Evaluating Large Language Models Trained on Code",
      "submitted_url": "https://arxiv.org/abs/2107.03374",
      "submitted_author_label": [
        "Chen, M.",
        "Tworek, J.",
        "Jun, H.",
        "and others"
      ],
      "submitted_date_label": "2021-07-07",
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-madaan-variance",
      "submitted_title": "Quantifying Variance in Evaluation Benchmarks",
      "submitted_url": "https://arxiv.org/abs/2406.10229",
      "submitted_author_label": [
        "Madaan, L.",
        "Singh, A. K.",
        "Schaeffer, R.",
        "Poulton, A.",
        "Koyejo, S.",
        "Stenetorp, P.",
        "Narang, S.",
        "Hupkes, D."
      ],
      "submitted_date_label": "2024-06-14",
      "verified_source_id": "madaan",
      "verified_title": "Quantifying Variance in Evaluation Benchmarks",
      "verified_authors": "Lovish Madaan et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-card-power",
      "submitted_title": "With Little Power Comes Great Responsibility",
      "submitted_url": "https://arxiv.org/abs/2010.06595",
      "submitted_author_label": [
        "Card, D.",
        "Henderson, P.",
        "Khandelwal, U.",
        "Jia, R.",
        "Mahowald, K.",
        "Jurafsky, D."
      ],
      "submitted_date_label": "2020-10-13",
      "verified_source_id": "card",
      "verified_title": "With Little Power Comes Great Responsibility",
      "verified_authors": "Dallas Card et al.",
      "verified_year": "2020",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-shankar-validators",
      "submitted_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
      "submitted_url": "https://arxiv.org/abs/2404.12272",
      "submitted_author_label": [
        "Shankar, S.",
        "Zamfirescu-Pereira, J. D.",
        "Hartmann, B.",
        "Parameswaran, A. G.",
        "Arawjo, I."
      ],
      "submitted_date_label": "2024-04-18",
      "verified_source_id": "shankar",
      "verified_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
      "verified_authors": "Shreya Shankar et al.",
      "verified_year": "2024",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "C",
      "submitted_source_id": "src-inspect",
      "submitted_title": "Inspect: Framework for Large Language Model Evaluations",
      "submitted_url": "https://inspect.aisi.org.uk/",
      "submitted_author_label": [
        "UK AI Security Institute",
        "Meridian Labs"
      ],
      "submitted_date_label": "2024-05",
      "verified_source_id": "inspect",
      "verified_title": "Inspect",
      "verified_authors": "UK AI Security Institute and Meridian Labs",
      "verified_year": "living documentation",
      "retrieval_date_utc": "2026-10-25",
      "verdict": "associated_source_record_confirmed",
      "note": "Minor shortened titles and preprint versus venue years are acceptable. Living-page publication years are access-year labels, not authenticated original publication dates."
    },
    {
      "cluster": "A",
      "article_file": "A/articles/agent-system-evaluation.md",
      "line": 33,
      "submitted_label": "Evaluate agent workflows",
      "submitted_url": "https://developers.openai.com/api/docs/guides/agent-evals",
      "verified_source_id": "agentdocs",
      "verified_title": "Evaluate agent workflows",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/agent-system-evaluation.md",
      "line": 43,
      "submitted_label": "Zheng et al., 2026",
      "submitted_url": "https://arxiv.org/abs/2609.08149",
      "verified_source_id": "zhengpro",
      "verified_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents",
      "verified_authors": "Pujun Zheng et al.",
      "verified_year": "2026",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "line": 33,
      "submitted_label": "Hendrycks et al., 2020",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "line": 33,
      "submitted_label": "Liang et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "line": 76,
      "submitted_label": "official pricing page",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/choosing-model-benchmark.md",
      "line": 84,
      "submitted_label": "Jimenez et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "line": 31,
      "submitted_label": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/datasets-holdouts-contamination.md",
      "line": 31,
      "submitted_label": "Xiao and Cheng, 2026",
      "submitted_url": "https://arxiv.org/abs/2609.02899",
      "verified_source_id": "xiao",
      "verified_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards",
      "verified_authors": "Xingyao Xiao and Yihong Cheng",
      "verified_year": "2026",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "line": 35,
      "submitted_label": "Zheng et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.07791",
      "verified_source_id": "position",
      "verified_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
      "verified_authors": "Lin Shi et al.",
      "verified_year": "2024 (revised 2025)",
      "verdict": "wrong_author_label",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/human-evaluation-model-judges.md",
      "line": 51,
      "submitted_label": "evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/latency-cost.md",
      "line": 29,
      "submitted_label": "official pricing",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/maintaining-evaluations.md",
      "line": 25,
      "submitted_label": "evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/metrics-rubrics.md",
      "line": 31,
      "submitted_label": "OpenAI evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/metrics-rubrics.md",
      "line": 45,
      "submitted_label": "Graders guide",
      "submitted_url": "https://developers.openai.com/api/docs/guides/graders",
      "verified_source_id": "graders",
      "verified_title": "Graders",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "line": 21,
      "submitted_label": "Hendrycks et al., 2020",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "line": 31,
      "submitted_label": "Liang et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "line": 39,
      "submitted_label": "Zheng et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.07791",
      "verified_source_id": "position",
      "verified_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge",
      "verified_authors": "Lin Shi et al.",
      "verified_year": "2024 (revised 2025)",
      "verdict": "wrong_author_label",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/public-benchmark-literacy.md",
      "line": 45,
      "submitted_label": "Rethinking Benchmark and Contamination",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "A",
      "article_file": "A/articles/repeatability-uncertainty.md",
      "line": 44,
      "submitted_label": "Bouthillier et al., 2021",
      "submitted_url": "https://arxiv.org/abs/2103.03098",
      "verified_source_id": "variance",
      "verified_title": "Accounting for Variance in Machine Learning Benchmarks",
      "verified_authors": "Xavier Bouthillier et al.",
      "verified_year": "2021",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "line": 21,
      "submitted_label": "Latency optimisation guidance",
      "submitted_url": "https://developers.openai.com/api/docs/guides/latency-optimization",
      "verified_source_id": "latency",
      "verified_title": "Latency optimization",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "line": 41,
      "submitted_label": "Official pricing, accessed 22 October 2026",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/ai-latency-cost-per-successful-task.md",
      "line": 51,
      "submitted_label": "Prompt caching documentation",
      "submitted_url": "https://developers.openai.com/api/docs/guides/prompt-caching",
      "verified_source_id": "cache",
      "verified_title": "Prompt caching",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "line": 33,
      "submitted_label": "MMLU paper",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/ai-model-benchmarking-guide.md",
      "line": 73,
      "submitted_label": "Evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "line": 39,
      "submitted_label": "NIST confidence interval guidance",
      "submitted_url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm",
      "verified_source_id": "wilson",
      "verified_title": "7.2.4.1. Confidence intervals",
      "verified_authors": "NIST/SEMATECH",
      "verified_year": "living reference",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "line": 43,
      "submitted_label": "Coverage interpretation in the SciPy documentation",
      "submitted_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
      "verified_source_id": "bootstrap",
      "verified_title": "scipy.stats.bootstrap",
      "verified_authors": "SciPy developers",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/benchmark-uncertainty-repeatability.md",
      "line": 51,
      "submitted_label": "Bootstrap reference",
      "submitted_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html",
      "verified_source_id": "bootstrap",
      "verified_title": "scipy.stats.bootstrap",
      "verified_authors": "SciPy developers",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/design-llm-evaluation-tasks.md",
      "line": 55,
      "submitted_label": "CheckList paper",
      "submitted_url": "https://aclanthology.org/2020.acl-main.442/",
      "verified_source_id": "checklist",
      "verified_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList",
      "verified_authors": "Marco Tulio Ribeiro et al.",
      "verified_year": "2020",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "line": 43,
      "submitted_label": "Original τ-bench paper",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md",
      "line": 73,
      "submitted_label": "τ-bench metric definition",
      "submitted_url": "https://arxiv.org/html/2406.12045v1#S3",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "line": 21,
      "submitted_label": "Common pitfalls",
      "submitted_url": "https://scikit-learn.org/stable/common_pitfalls.html",
      "verified_source_id": "leak",
      "verified_title": "Common pitfalls and recommended practices",
      "verified_authors": "scikit-learn developers",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "line": 23,
      "submitted_label": "Detecting Pretraining Data from Large Language Models",
      "submitted_url": "https://arxiv.org/abs/2310.16789",
      "verified_source_id": "shi",
      "verified_title": "Detecting Pretraining Data from Large Language Models",
      "verified_authors": "Weijia Shi et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "line": 31,
      "submitted_label": "Datasheets paper",
      "submitted_url": "https://arxiv.org/abs/1803.09010",
      "verified_source_id": "datasheet",
      "verified_title": "Datasheets for Datasets",
      "verified_authors": "Timnit Gebru et al.",
      "verified_year": "2018 (CACM 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md",
      "line": 65,
      "submitted_label": "Proving Test Set Contamination in Black Box Language Models",
      "submitted_url": "https://arxiv.org/abs/2310.17623",
      "verified_source_id": "oren",
      "verified_title": "Proving Test Set Contamination in Black Box Language Models",
      "verified_authors": "Yonatan Oren et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "line": 33,
      "submitted_label": "Prompt-format study",
      "submitted_url": "https://arxiv.org/abs/2310.11324",
      "verified_source_id": "sclar",
      "verified_title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting",
      "verified_authors": "Melanie Sclar et al.",
      "verified_year": "2023 (ICLR 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "line": 51,
      "submitted_label": "Reasoning model documentation",
      "submitted_url": "https://developers.openai.com/api/docs/guides/reasoning",
      "verified_source_id": "reason",
      "verified_title": "Reasoning models",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/fair-ai-model-comparisons.md",
      "line": 75,
      "submitted_label": "Harness repository",
      "submitted_url": "https://github.com/EleutherAI/lm-evaluation-harness",
      "verified_source_id": "harness",
      "verified_title": "Language Model Evaluation Harness",
      "verified_authors": "EleutherAI",
      "verified_year": "living repository",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/human-reviewers-llm-judges.md",
      "line": 49,
      "submitted_label": "Original judge study",
      "submitted_url": "https://arxiv.org/abs/2306.05685",
      "verified_source_id": "judge",
      "verified_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "verified_authors": "Lianmin Zheng et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/llm-evaluation-metrics-rubrics.md",
      "line": 41,
      "submitted_label": "Scikit-learn metric reference",
      "submitted_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
      "verified_source_id": "metrics",
      "verified_title": "precision_recall_fscore_support",
      "verified_authors": "scikit-learn developers",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/llm-evaluation-metrics-rubrics.md",
      "line": 47,
      "submitted_label": "Averaging definitions",
      "submitted_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html",
      "verified_source_id": "metrics",
      "verified_title": "precision_recall_fscore_support",
      "verified_authors": "scikit-learn developers",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/maintain-llm-evaluation-suite.md",
      "line": 23,
      "submitted_label": "Evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "line": 23,
      "submitted_label": "Original HumanEval paper",
      "submitted_url": "https://arxiv.org/abs/2107.03374",
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "line": 35,
      "submitted_label": "HumanEval evaluation framework",
      "submitted_url": "https://arxiv.org/html/2107.03374v2#S2.SS1",
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "line": 43,
      "submitted_label": "Official SWE-bench leaderboards, accessed 22 October 2026",
      "submitted_url": "https://www.swebench.com/",
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "B",
      "article_file": "B/articles/read-ai-benchmark-leaderboards.md",
      "line": 53,
      "submitted_label": "HELM paper",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 28,
      "submitted_label": "Kapoor et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2407.01502",
      "verified_source_id": "kapoor",
      "verified_title": "AI Agents That Matter",
      "verified_authors": "Sayash Kapoor et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 36,
      "submitted_label": "Yang et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 38,
      "submitted_label": "Deng et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2311.09783",
      "verified_source_id": "deng",
      "verified_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
      "verified_authors": "Chunyuan Deng et al.",
      "verified_year": "2023 (NAACL 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 40,
      "submitted_label": "Cheng et al., 2025",
      "submitted_url": "https://arxiv.org/abs/2502.14425",
      "verified_source_id": "cheng",
      "verified_title": "A Survey on Data Contamination for Large Language Models",
      "verified_authors": "Yuxing Cheng, Yi Chang and Yuan Wu",
      "verified_year": "2025",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 44,
      "submitted_label": "Jain et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2403.07974",
      "verified_source_id": "live",
      "verified_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
      "verified_authors": "Naman Jain et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/contamination-leakage-and-holdouts.md",
      "line": 60,
      "submitted_label": "Singh et al., 2025",
      "submitted_url": "https://arxiv.org/abs/2504.20879",
      "verified_source_id": "illusion",
      "verified_title": "The Leaderboard Illusion",
      "verified_authors": "Shivalika Singh et al.",
      "verified_year": "2025",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "line": 26,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "line": 102,
      "submitted_label": "Shankar et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2404.12272",
      "verified_source_id": "shankar",
      "verified_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
      "verified_authors": "Shreya Shankar et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "line": 114,
      "submitted_label": "Miller, 2024",
      "submitted_url": "https://arxiv.org/abs/2411.00640",
      "verified_source_id": "miller",
      "verified_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "verified_authors": "Evan Miller",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "line": 114,
      "submitted_label": "Card et al., 2020",
      "submitted_url": "https://arxiv.org/abs/2010.06595",
      "verified_source_id": "card",
      "verified_title": "With Little Power Comes Great Responsibility",
      "verified_authors": "Dallas Card et al.",
      "verified_year": "2020",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md",
      "line": 156,
      "submitted_label": "Reuel et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2411.12990",
      "verified_source_id": "better",
      "verified_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "verified_authors": "Anka Reuel et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 31,
      "submitted_label": "Kapoor et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2407.01502",
      "verified_source_id": "kapoor",
      "verified_title": "AI Agents That Matter",
      "verified_authors": "Sayash Kapoor et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 37,
      "submitted_label": "Jimenez et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 37,
      "submitted_label": "SWE-bench leaderboards",
      "submitted_url": "https://www.swebench.com/",
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 39,
      "submitted_label": "Yao et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 41,
      "submitted_label": "Mialon et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2311.12983",
      "verified_source_id": "gaia",
      "verified_title": "GAIA: a benchmark for General AI Assistants",
      "verified_authors": "Grégoire Mialon et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 43,
      "submitted_label": "Jain et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2403.07974",
      "verified_source_id": "live",
      "verified_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
      "verified_authors": "Naman Jain et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 73,
      "submitted_label": "OpenAI, agent workflows",
      "submitted_url": "https://developers.openai.com/api/docs/guides/agent-evals",
      "verified_source_id": "agentdocs",
      "verified_title": "Evaluate agent workflows",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-agents-and-tool-use.md",
      "line": 85,
      "submitted_label": "UK AI Security Institute, Inspect",
      "submitted_url": "https://inspect.aisi.org.uk/",
      "verified_source_id": "inspect",
      "verified_title": "Inspect",
      "verified_authors": "UK AI Security Institute and Meridian Labs",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 22,
      "submitted_label": "Hendrycks et al., 2021",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 22,
      "submitted_label": "Srivastava et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2206.04615",
      "verified_source_id": "big",
      "verified_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "verified_authors": "Aarohi Srivastava et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 22,
      "submitted_label": "Liang et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 26,
      "submitted_label": "Chiang et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2403.04132",
      "verified_source_id": "arena",
      "verified_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
      "verified_authors": "Wei-Lin Chiang et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 26,
      "submitted_label": "Singh et al., 2025",
      "submitted_url": "https://arxiv.org/abs/2504.20879",
      "verified_source_id": "illusion",
      "verified_title": "The Leaderboard Illusion",
      "verified_authors": "Shivalika Singh et al.",
      "verified_year": "2025",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 26,
      "submitted_label": "Reuel et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2411.12990",
      "verified_source_id": "better",
      "verified_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "verified_authors": "Anka Reuel et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 34,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 36,
      "submitted_label": "Jimenez et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 36,
      "submitted_label": "SWE-bench leaderboards",
      "submitted_url": "https://www.swebench.com/",
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 36,
      "submitted_label": "Mialon et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2311.12983",
      "verified_source_id": "gaia",
      "verified_title": "GAIA: a benchmark for General AI Assistants",
      "verified_authors": "Grégoire Mialon et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 36,
      "submitted_label": "Yao et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 64,
      "submitted_label": "Yang et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2311.04850",
      "verified_source_id": "yang",
      "verified_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples",
      "verified_authors": "Shuo Yang et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 68,
      "submitted_label": "Miller, 2024",
      "submitted_url": "https://arxiv.org/abs/2411.00640",
      "verified_source_id": "miller",
      "verified_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "verified_authors": "Evan Miller",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md",
      "line": 72,
      "submitted_label": "Kapoor et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2407.01502",
      "verified_source_id": "kapoor",
      "verified_title": "AI Agents That Matter",
      "verified_authors": "Sayash Kapoor et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "line": 38,
      "submitted_label": "Shankar et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2404.12272",
      "verified_source_id": "shankar",
      "verified_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
      "verified_authors": "Shreya Shankar et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "line": 72,
      "submitted_label": "Zheng et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2306.05685",
      "verified_source_id": "judge",
      "verified_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "verified_authors": "Lianmin Zheng et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/human-evaluation-and-model-judges.md",
      "line": 76,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "line": 45,
      "submitted_label": "Reuel et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2411.12990",
      "verified_source_id": "better",
      "verified_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "verified_authors": "Anka Reuel et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "line": 75,
      "submitted_label": "Singh et al., 2025",
      "submitted_url": "https://arxiv.org/abs/2504.20879",
      "verified_source_id": "illusion",
      "verified_title": "The Leaderboard Illusion",
      "verified_authors": "Shivalika Singh et al.",
      "verified_year": "2025",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/interpreting-results-and-maintaining-evals.md",
      "line": 112,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 20,
      "submitted_label": "Kapoor et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2407.01502",
      "verified_source_id": "kapoor",
      "verified_title": "AI Agents That Matter",
      "verified_authors": "Sayash Kapoor et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 36,
      "submitted_label": "OpenAI, latency optimization",
      "submitted_url": "https://developers.openai.com/api/docs/guides/latency-optimization",
      "verified_source_id": "latency",
      "verified_title": "Latency optimization",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 45,
      "submitted_label": "OpenAI, reasoning models",
      "submitted_url": "https://developers.openai.com/api/docs/guides/reasoning",
      "verified_source_id": "reason",
      "verified_title": "Reasoning models",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 78,
      "submitted_label": "OpenAI pricing",
      "submitted_url": "https://developers.openai.com/api/docs/pricing",
      "verified_source_id": "price",
      "verified_title": "Pricing",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 78,
      "submitted_label": "Anthropic pricing",
      "submitted_url": "https://claude.com/pricing",
      "verified_source_id": "claude",
      "verified_title": "Pricing",
      "verified_authors": "Anthropic",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/latency-token-usage-and-cost.md",
      "line": 78,
      "submitted_label": "Google Gemini pricing",
      "submitted_url": "https://ai.google.dev/gemini-api/docs/pricing",
      "verified_source_id": "gemini",
      "verified_title": "Gemini Developer API pricing",
      "verified_authors": "Google",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 30,
      "submitted_label": "Chen et al., 2021",
      "submitted_url": "https://arxiv.org/abs/2107.03374",
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 32,
      "submitted_label": "Yao et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 34,
      "submitted_label": "Jimenez et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2310.06770",
      "verified_source_id": "swe",
      "verified_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "verified_authors": "Carlos E. Jimenez et al.",
      "verified_year": "2023 (ICLR 2024)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 38,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 44,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 54,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/objective-metrics-and-rubrics.md",
      "line": 92,
      "submitted_label": "Zheng et al., 2023",
      "submitted_url": "https://arxiv.org/abs/2306.05685",
      "verified_source_id": "judge",
      "verified_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "verified_authors": "Lianmin Zheng et al.",
      "verified_year": "2023",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "line": 14,
      "submitted_label": "OpenAI, evaluation best practices",
      "submitted_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices",
      "verified_source_id": "eval",
      "verified_title": "Evaluation best practices",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "line": 74,
      "submitted_label": "UK AI Security Institute, Inspect",
      "submitted_url": "https://inspect.aisi.org.uk/",
      "verified_source_id": "inspect",
      "verified_title": "Inspect",
      "verified_authors": "UK AI Security Institute and Meridian Labs",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "line": 100,
      "submitted_label": "SWE-bench leaderboards",
      "submitted_url": "https://www.swebench.com/",
      "verified_source_id": "swesite",
      "verified_title": "SWE-bench Leaderboards",
      "verified_authors": "SWE-bench team",
      "verified_year": "living maintainer page",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/prompt-tool-and-configuration-fairness.md",
      "line": 110,
      "submitted_label": "OpenAI, reasoning models",
      "submitted_url": "https://developers.openai.com/api/docs/guides/reasoning",
      "verified_source_id": "reason",
      "verified_title": "Reasoning models",
      "verified_authors": "OpenAI",
      "verified_year": "living documentation",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 22,
      "submitted_label": "Hendrycks et al., 2021",
      "submitted_url": "https://arxiv.org/abs/2009.03300",
      "verified_source_id": "mmlu",
      "verified_title": "Measuring Massive Multitask Language Understanding",
      "verified_authors": "Dan Hendrycks et al.",
      "verified_year": "2020 (ICLR 2021)",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 22,
      "submitted_label": "Srivastava et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2206.04615",
      "verified_source_id": "big",
      "verified_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "verified_authors": "Aarohi Srivastava et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 24,
      "submitted_label": "Liang et al., 2022",
      "submitted_url": "https://arxiv.org/abs/2211.09110",
      "verified_source_id": "helm",
      "verified_title": "Holistic Evaluation of Language Models",
      "verified_authors": "Percy Liang et al.",
      "verified_year": "2022",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 38,
      "submitted_label": "Chiang et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2403.04132",
      "verified_source_id": "arena",
      "verified_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
      "verified_authors": "Wei-Lin Chiang et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 40,
      "submitted_label": "Singh et al., 2025",
      "submitted_url": "https://arxiv.org/abs/2504.20879",
      "verified_source_id": "illusion",
      "verified_title": "The Leaderboard Illusion",
      "verified_authors": "Shivalika Singh et al.",
      "verified_year": "2025",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md",
      "line": 42,
      "submitted_label": "Reuel et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2411.12990",
      "verified_source_id": "better",
      "verified_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
      "verified_authors": "Anka Reuel et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "line": 28,
      "submitted_label": "Madaan et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.10229",
      "verified_source_id": "madaan",
      "verified_title": "Quantifying Variance in Evaluation Benchmarks",
      "verified_authors": "Lovish Madaan et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "line": 42,
      "submitted_label": "Miller, 2024",
      "submitted_url": "https://arxiv.org/abs/2411.00640",
      "verified_source_id": "miller",
      "verified_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
      "verified_authors": "Evan Miller",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "line": 74,
      "submitted_label": "Chen et al., 2021",
      "submitted_url": "https://arxiv.org/abs/2107.03374",
      "verified_source_id": "code",
      "verified_title": "Evaluating Large Language Models Trained on Code",
      "verified_authors": "Mark Chen et al.",
      "verified_year": "2021",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "line": 76,
      "submitted_label": "Yao et al., 2024",
      "submitted_url": "https://arxiv.org/abs/2406.12045",
      "verified_source_id": "tau",
      "verified_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
      "verified_authors": "Shunyu Yao et al.",
      "verified_year": "2024",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    },
    {
      "cluster": "C",
      "article_file": "C/articles/repeatability-variance-and-uncertainty.md",
      "line": 82,
      "submitted_label": "Card et al., 2020",
      "submitted_url": "https://arxiv.org/abs/2010.06595",
      "verified_source_id": "card",
      "verified_title": "With Little Power Comes Great Responsibility",
      "verified_authors": "Dallas Card et al.",
      "verified_year": "2020",
      "verdict": "associated_primary_record_confirmed",
      "retrieval_date_utc": "2026-10-25"
    }
  ],
  "bibliography_note": "All distinct cited source URLs matched to live original/official records. Bibliographic labels checked against author/date/title records, accepting preprint and venue years. Exact author initials were not independently disambiguated beyond source records; Inspect’s 2024 citation year was not established by its current homepage. C’s contamination survey is secondary synthesis and should be replaced with originals for a strict primary-only brief.",
  "verification_limits": [
    "No archived provider documentation/pricing snapshot from 22 or 25 October was independently obtained; today’s live pages corroborate or qualify the submitted claims, without proving historical unchangedness.",
    "No underlying benchmark experiment, closed training-data audit, account-specific availability or invoice/billing reconciliation was performed.",
    "General process advice, privacy/legal implications, every uncited rhetorical assertion and every possible failure case were not exhaustively fact-checked.",
    "Selected paper passages/abstracts were read to verify their used claims; entire referenced literature was not reproduced or audited.",
    "Tool retrieval of a SQuAD script failed and was not used as evidence. F1 critique follows the checked precision/recall definition and an explicit counterexample.",
    "No fabricated citation or empirical result was established; wrong authors, version labels, unsupported guidance and hypothetical arithmetic are distinguished."
  ],
  "unchecked_claims_by_article": {
    "A/articles/agent-system-evaluation.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/build-task-eval.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/choosing-model-benchmark.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/datasets-holdouts-contamination.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/fair-model-comparisons.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/human-evaluation-model-judges.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/latency-cost.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/maintaining-evaluations.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/metrics-rubrics.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/public-benchmark-literacy.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "A/articles/repeatability-uncertainty.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/ai-latency-cost-per-successful-task.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/ai-model-benchmarking-guide.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/benchmark-uncertainty-repeatability.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/design-llm-evaluation-tasks.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/evaluate-ai-agents-tools-outcomes.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/evaluation-dataset-leakage-holdouts.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/fair-ai-model-comparisons.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/human-reviewers-llm-judges.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/llm-evaluation-metrics-rubrics.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/maintain-llm-evaluation-suite.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "B/articles/read-ai-benchmark-leaderboards.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/contamination-leakage-and-holdouts.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/designing-evaluation-tasks-and-datasets.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/evaluating-agents-and-tool-use.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/evaluating-ai-models-and-agent-systems.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/human-evaluation-and-model-judges.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/interpreting-results-and-maintaining-evals.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/latency-token-usage-and-cost.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/objective-metrics-and-rubrics.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/prompt-tool-and-configuration-fairness.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/public-benchmarks-vs-workload-evaluation.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation.",
    "C/articles/repeatability-variance-and-uncertainty.md": "Claims not individually listed in claim_records remain outside this targeted factual audit; full prose was read for editorial quality. Recommendations are not asserted to have experimental validation."
  },
  "review_local_calendar_date": "2026-10-26",
  "local_timezone": "Australia/Melbourne",
  "utc_date_basis": "Live clock and system date checked: 2026-10-25T14:33:52Z. All source retrievals in this review occurred on 25 October UTC / 26 October Melbourne."
}