{ "reviewed_at_utc": "2026-10-28T02:19:48.851565+00:00", "verification_date": "2026-10-28", "research_date_policy": "Candidate dates remain separate. Live pages corroborate current content; historical access or account-specific invoices cannot be proven without an archived snapshot. No discrepancy alone establishes dishonesty.", "method": "Full reading of44drafts; purposive checks of substantive methodological/source claims, all discovered numerical benchmark results and material changing price/specification/schedule claims. No experiments reproduced. Arithmetic recomputed independently. Source summaries/locations are evidence locators, not full-paper audits.", "verdict_policy": { "inaccurate_bibliographic_author": "Real supporting work, wrong name attached; not fabricated evidence.", "unsupported_generalisation": "Source does not establish the claim; not evidence that the cited work is fabricated.", "supported_inference": "Reviewer checks the methodological inference; source supplies underlying principle, not verbatim local recommendation.", "arithmetic_verified": "Labelled hypothetical inputs; arithmetic verified, no measured outcome.", "material_accounting_ambiguity": "Operational formula could undercount if followed literally; repair before publication." }, "sources": { "agentbench": { "source_title": "AgentBench: Evaluating LLMs as Agents", "source_url": "https://arxiv.org/abs/2308.03688", "author_or_maintainer": "Xiao Liu et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Eight interactive environments; reasoning and decision-making are assessed.", "type": "original paper" }, "helm": { "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "author_or_maintainer": "Percy Liang et al.", "year": 2022, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Seven metrics; 16 core and 26 targeted scenarios; 30 models; prior coverage 17.9%, raised to 96.0%; standardised conditions.", "type": "original paper" }, "balloccu": { "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_url": "https://arxiv.org/abs/2402.03927", "author_or_maintainer": "Simone Balloccu et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Indirect exposure and reproducibility problems; exposure does not establish memorisation of an individual item.", "type": "original paper" }, "position": { "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "author_or_maintainer": "Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma, Soroush Vosoughi", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Position effects vary by judge, candidate and task; latest version November 2025.", "type": "original paper" }, "mmlu": { "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "author_or_maintainer": "Dan Hendrycks et al.", "year": "2020 preprint; ICLR 2021", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "57 tasks cover academic/professional subjects; historical models lagged expert accuracy.", "type": "original paper" }, "bigbench": { "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "source_url": "https://arxiv.org/abs/2206.04615", "author_or_maintainer": "Aarohi Srivastava et al.", "year": 2022, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "204 tasks, 450 authors, 132 institutions, expert raters; scale improves performance/calibration; brittle metrics affect some breakthroughs.", "type": "original paper" }, "arena": { "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "source_url": "https://arxiv.org/abs/2403.04132", "author_or_maintainer": "Wei-Lin Chiang et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Pairwise crowdsourced preferences; over 240K votes at paper date; crowd/expert agreement.", "type": "original paper" }, "illusion": { "source_title": "The Leaderboard Illusion", "source_url": "https://arxiv.org/abs/2504.20879", "author_or_maintainer": "Shivalika Singh et al.", "year": 2025, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Private testing and selective reporting; Google/OpenAI data shares estimated 19.2%/20.4%, 83 open models 29.7%.", "type": "original paper" }, "betterbench": { "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_url": "https://arxiv.org/abs/2411.12990", "author_or_maintainer": "Anka Reuel et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "24 benchmarks assessed against 46 practices; significance and replication reporting gaps.", "type": "original paper" }, "swepaper": { "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_url": "https://arxiv.org/abs/2310.06770", "author_or_maintainer": "Carlos E. Jimenez et al.", "year": "2023 preprint; ICLR 2024", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "2,294 issues across 12 Python repositories; original best Claude 2 result 1.96%.", "type": "original paper" }, "gaia": { "source_title": "GAIA: a benchmark for General AI Assistants", "source_url": "https://arxiv.org/abs/2311.12983", "author_or_maintainer": "Gr\u00e9goire Mialon et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "466 questions; 92% humans versus 15% GPT-4 with plugins; 300 answers held back.", "type": "original paper" }, "tau": { "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "author_or_maintainer": "Shunyu Yao et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Simulated users, APIs, policy and database-state reward; historical success <50%, retail pass^8 <25%.", "type": "original paper" }, "taumetric": { "source_title": "\u03c4-bench, section 3", "source_url": "https://arxiv.org/html/2406.12045v1#S3", "author_or_maintainer": "Shunyu Yao et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Section 3, reward and pass^k paragraphs", "evidence_summary": "All k i.i.d. successes averaged by task; estimator C(c,k)/C(n,k); final state alone can miss permission breaches.", "type": "original paper" }, "yang": { "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "author_or_maintainer": "Shuo Yang et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Paraphrases evade n-gram filtering; 8\u201318% HumanEval overlap; contaminated 13B model can match much larger-model benchmark performance.", "type": "original paper" }, "deng": { "source_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "source_url": "https://arxiv.org/abs/2311.09783", "author_or_maintainer": "Chunyuan Deng et al.", "year": "2023 preprint; NAACL 2024", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "TS-Guessing masks wrong options; missing-option exact match 52% ChatGPT and 57% GPT-4 on MMLU.", "type": "original paper" }, "cheng": { "source_title": "A Survey on Data Contamination for Large Language Models", "source_url": "https://arxiv.org/abs/2502.14425", "author_or_maintainer": "Yuxing Cheng, Yi Chang, Yuan Wu", "year": 2025, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Updating, rewriting, prevention; detection classified by white/grey/black-box access.", "type": "original paper" }, "livecode": { "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "source_url": "https://arxiv.org/abs/2403.07974", "author_or_maintainer": "Naman Jain et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "v2 has 400 problems from May 2023\u2013May 2024; contests on LeetCode, AtCoder, Codeforces.", "type": "original paper" }, "livecodev1": { "source_title": "LiveCodeBench, initial version", "source_url": "https://arxiv.org/abs/2403.07974v1", "author_or_maintainer": "Naman Jain et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "v1 abstract, submitted 12 March 2024", "evidence_summary": "v1 has 400 problems from May 2023\u2013February 2024, not through May 2024.", "type": "original paper" }, "madaan": { "source_title": "Quantifying Variance in Evaluation Benchmarks", "source_url": "https://arxiv.org/abs/2406.10229", "author_or_maintainer": "Lovish Madaan et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Seed/training variance and choice-versus-completion framing studied; format changes can reduce variance for smaller models.", "type": "original paper" }, "miller": { "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_url": "https://arxiv.org/abs/2411.00640", "author_or_maintainer": "Evan Miller", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Question-level analysis; paired differences; grouping and repeated outputs require suitable variance treatment.", "type": "original paper" }, "millerfull": { "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "author_or_maintainer": "Evan Miller", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Sections 2\u20135; table 4; equations 1\u20139", "evidence_summary": "SE=sd/sqrt(n); clustered example ratio 3.05; conditional variance divides by K; illustrative reductions 1/3,1/2,limit2/3; 969-question worked assumption. Paired covariance formula supplies independent recalculation.", "type": "original paper" }, "card": { "source_title": "With Little Power Comes Great Responsibility", "source_url": "https://arxiv.org/abs/2010.06595", "author_or_maintainer": "Dallas Card et al.", "year": 2020, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Underpowered NLP studies; typical 2,000-sentence MT sets approximately 75% power for 1 BLEU under assumptions.", "type": "original paper" }, "shankar": { "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_url": "https://arxiv.org/abs/2404.12272", "author_or_maintainer": "Shreya Shankar et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Criteria drift: judging examples can change the evaluation criteria.", "type": "original paper" }, "kapoor": { "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "author_or_maintainer": "Sayash Kapoor et al.", "year": 2024, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Accuracy-only focus, cost optimisation, weak holdouts and reproducibility; downstream/model-developer needs differ.", "type": "original paper" }, "mtbench": { "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_url": "https://arxiv.org/abs/2306.05685", "author_or_maintainer": "Lianmin Zheng et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Position, verbosity, self-enhancement biases and reasoning limits; >80% human agreement in tested settings.", "type": "original paper" }, "humaneval": { "source_title": "Evaluating Large Language Models Trained on Code", "source_url": "https://arxiv.org/abs/2107.03374", "author_or_maintainer": "Mark Chen et al.", "year": 2021, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "HumanEval tests program functional correctness from docstrings; repeated sampling and executable verification.", "type": "original paper" }, "passk": { "source_title": "Evaluating Large Language Models Trained on Code, section 2.1", "source_url": "https://arxiv.org/html/2107.03374v2#S2.SS1", "author_or_maintainer": "Mark Chen et al.", "year": 2021, "retrieved_at_date": "2026-10-28", "inspected_location": "Section 2.1, equation 1; Appendix A", "evidence_summary": "For n>=k and c passing samples, use 1-C(n-c,k)/C(n,k); plug-in power estimator is biased.", "type": "original paper" }, "shi": { "source_title": "Detecting Pretraining Data from Large Language Models", "source_url": "https://arxiv.org/abs/2310.16789", "author_or_maintainer": "Weijia Shi et al.", "year": "2023 preprint; revised 2024", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Detection with black-box model access and no training corpus; method needs token probability access.", "type": "original paper" }, "oren": { "source_title": "Proving Test Set Contamination in Black Box Language Models", "source_url": "https://arxiv.org/abs/2310.17623", "author_or_maintainer": "Yonatan Oren et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Canonical-versus-shuffled likelihood test assumes exchangeable benchmark ordering.", "type": "original paper" }, "datasheets": { "source_title": "Datasheets for Datasets", "source_url": "https://arxiv.org/abs/1803.09010", "author_or_maintainer": "Timnit Gebru et al.", "year": "2018 preprint; CACM 2021", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Document motivation, composition, collection and recommended use.", "type": "original paper" }, "sclar": { "source_title": "Quantifying Language Models\u2019 Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting", "source_url": "https://arxiv.org/abs/2310.11324", "author_or_maintainer": "Melanie Sclar et al.", "year": "2023 preprint; revised 2024", "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Meaning-preserving formats affect studied few-shot models; preferred formats transfer weakly.", "type": "original paper" }, "xiao": { "source_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards", "source_url": "https://arxiv.org/abs/2609.02899", "author_or_maintainer": "Xingyao Xiao and Yihong Cheng", "year": 2026, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Paraphrase-controlled analysis distinguishes score inflation from rank changes; conclusion specific to tested models/benchmarks.", "type": "original paper" }, "pro": { "source_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents", "source_url": "https://arxiv.org/abs/2609.08149", "author_or_maintainer": "Pujun Zheng et al.", "year": 2026, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "SWE-Bench Pro leakage/reward hacking and task quality examined; verified version proposed.", "type": "original paper" }, "bouthillier": { "source_title": "Accounting for Variance in Machine Learning Benchmarks", "source_url": "https://arxiv.org/abs/2103.03098", "author_or_maintainer": "Xavier Bouthillier et al.", "year": 2021, "retrieved_at_date": "2026-10-28", "inspected_location": "Title, authors, submission history and abstract", "evidence_summary": "Data sampling, initialisation and hyperparameter variance affect comparisons; methods evaluated across five tasks.", "type": "original paper" }, "checklist": { "source_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList", "source_url": "https://aclanthology.org/2020.acl-main.442/", "author_or_maintainer": "Marco Tulio Ribeiro, Tongshuang Wu, Carlos Guestrin, Sameer Singh", "year": 2020, "retrieved_at_date": "2026-10-28", "inspected_location": "Abstract and bibliographic record", "evidence_summary": "Matrix of linguistic capabilities and test types; behavioural tests complement held-out accuracy.", "type": "original paper" }, "eval": { "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Evals tips; design; human/model grading; architecture; transition notice", "evidence_summary": "Task-specific data, human calibration, continuous evaluation, typical/edge/adversarial cases; pairwise or criteria-based grading; Evals read-only 31 October and shutdown 30 November 2026. Recommendations require local validation.", "type": "official documentation or maintainer page" }, "reasoning": { "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Reasoning effort; managing reasoning tokens", "evidence_summary": "Reasoning tokens billed as output and use context; effort is model-dependent and adaptive; 25,000-token initial buffer recommended.", "type": "official documentation or maintainer page" }, "latency": { "source_title": "Latency optimization", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Generate fewer tokens; fewer input tokens; make users wait less", "evidence_summary": "Output-halving ~50% latency is a heuristic; input-halving 1\u20135% for ordinary contexts; streaming reduces visible waiting.", "type": "official documentation or maintainer page" }, "pricing": { "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Token tables and tool charges", "evidence_summary": "Standard Astra $10/$50/cache $1; Luna $0.10/$0.50/cache $0.01 per million; batch $5/$25 and $0.05/$0.25; write categories and tools separate.", "type": "official documentation or maintainer page" }, "caching": { "source_title": "Prompt caching", "source_url": "https://developers.openai.com/api/docs/guides/prompt-caching", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Rendered-prefix match, lookup boundaries, gotchas", "evidence_summary": "Matching eligible rendered prefix and lookup endpoint required; eligibility varies by model; shared text alone does not establish a hit.", "type": "official documentation or maintainer page" }, "agentevals": { "source_title": "Evaluate agent workflows", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Start with traces; datasets and repeatable evals", "evidence_summary": "Trace grading covers model/tool calls, guardrails and handoffs; datasets support repeatability.", "type": "official documentation or maintainer page" }, "graders": { "source_title": "Graders", "source_url": "https://developers.openai.com/api/docs/guides/graders", "author_or_maintainer": "OpenAI", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Types; best practices; grader hacking", "evidence_summary": "String, similarity, model and Python graders; compare with known-quality human labels; inspect reward hacking.", "type": "official documentation or maintainer page" }, "claude": { "source_title": "Plans and pricing", "source_url": "https://claude.com/pricing", "author_or_maintainer": "Anthropic", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Latest models, Haiku prompt-length tiers", "evidence_summary": "Fable $10/$50/read $0.25; Haiku $0.10/$0.50/read $0.01 up to 100K prompt tokens, $0.50/$2.50/read $0.05 above 100K.", "type": "official documentation or maintainer page" }, "claudedetail": { "source_title": "Pricing", "source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "author_or_maintainer": "Anthropic", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Model, batch and long-context pricing", "evidence_summary": "Batch halves input/output rates; Haiku higher prompt-length tier applies per request including cached tokens.", "type": "official documentation or maintainer page" }, "gemini": { "source_title": "Gemini Developer API pricing", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "author_or_maintainer": "Google", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Gemini 4 Argon standard/batch tables", "evidence_summary": "Argon standard $2/$10/cache $0.10 until 30 November 2026, then $4/$20/$0.20; batch $1/$5 then $2/$10; thinking billed in output.", "type": "official documentation or maintainer page" }, "sweoriginal": { "source_title": "SWE-bench", "source_url": "https://www.swebench.com/original.html", "author_or_maintainer": "SWE-bench maintainers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Overview", "evidence_summary": "Real issues/PRs; repository environments and fail-to-pass tests.", "type": "official documentation or maintainer page" }, "sweguide": { "source_title": "Evaluation Guide", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "author_or_maintainer": "SWE-bench maintainers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Understanding Evaluation Results, count definitions", "evidence_summary": "Resolved means required tests pass; unresolved differs from missing predictions, errors and infrastructure failures.", "type": "official documentation or maintainer page" }, "sweboard": { "source_title": "SWE-bench Leaderboards", "source_url": "https://www.swebench.com/", "author_or_maintainer": "SWE-bench maintainers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "View descriptions", "evidence_summary": "Verified/Bash Only 500; common mini-SWE-agent; Lite 300; Multilingual 300 from 42 repositories and nine languages.", "type": "official documentation or maintainer page" }, "skleak": { "source_title": "Common pitfalls and recommended practices", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "author_or_maintainer": "scikit-learn developers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Data leakage", "evidence_summary": "Unavailable-at-prediction information and test-based choices leak; split before fitting transformations.", "type": "official documentation or maintainer page" }, "skmetrics": { "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "author_or_maintainer": "scikit-learn developers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Definitions; average parameter", "evidence_summary": "Precision=TP/(TP+FP); recall=TP/(TP+FN); F1 harmonic mean; macro unweighted; micro global counts; weighted uses support.", "type": "official documentation or maintainer page" }, "bootstrap": { "source_title": "scipy.stats.bootstrap", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "author_or_maintainer": "SciPy developers", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Introduction; paired parameter; warnings", "evidence_summary": "Resample with replacement; paired uses shared indices; repeated-sampling coverage; percentile available with limitations.", "type": "official documentation or maintainer page" }, "nist": { "source_title": "7.2.4.1. Confidence intervals", "source_url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm", "author_or_maintainer": "NIST/SEMATECH", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Wilson formulas", "evidence_summary": "Wilson interval follows score-test inversion for binomial proportion.", "type": "official documentation or maintainer page" }, "nistexact": { "source_title": "Exact Binomial Confidence Limits", "source_url": "https://itl.nist.gov/div898/software/dataplot/refman2/auxillar/exacbici.htm", "author_or_maintainer": "NIST", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Description", "evidence_summary": "Normal approximation can fail for small samples or few failures; exact method offered.", "type": "official documentation or maintainer page" }, "inspect": { "source_title": "Inspect", "source_url": "https://inspect.aisi.org.uk/", "author_or_maintainer": "UK AI Security Institute / Meridian Labs", "year": "living page; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "Overview, components and sandbox sections", "evidence_summary": "Composable datasets, solvers, scorers, agents, tools and sandboxes.", "type": "official documentation or maintainer page" }, "harness": { "source_title": "Language Model Evaluation Harness", "source_url": "https://github.com/EleutherAI/lm-evaluation-harness", "author_or_maintainer": "EleutherAI", "year": "living repository; accessed 2026", "retrieved_at_date": "2026-10-28", "inspected_location": "README Overview", "evidence_summary": "Common framework with tasks and model arguments; framework use does not establish fairness.", "type": "official documentation or maintainer page" }, "bigbenchjournal": { "source_title": "Beyond the Imitation Game, journal version", "source_url": "https://joshrule.com/files/srivastava2023beyond.pdf", "author_or_maintainer": "Aarohi Srivastava et al.", "year": 2023, "retrieved_at_date": "2026-10-28", "inspected_location": "Page1 publication header", "evidence_summary": "Journal header states TMLR May2023; preprint2022 remains valid.", "type": "official documentation or maintainer page" } }, "claims": [ { "claim_id": "A-001", "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "AgentBench evaluates interactive environments" }, "claim_type": "factual_or_methodological_claim", "source_id": "agentbench", "source_title": "AgentBench: Evaluating LLMs as Agents", "source_url": "https://arxiv.org/abs/2308.03688", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Interactive environment scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-002", "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "HELM uses multiple dimensions including efficiency and robustness" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Named dimensions and scenario method match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-003", "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "SWE-bench distinguishes resolved, unresolved, incomplete and errors" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweguide", "source_title": "Evaluation Guide", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_location": "Understanding Evaluation Results, count definitions", "support_or_contradiction": "Count definitions match; required tests define resolved.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-004", "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Exposure routes do not prove individual memorisation" }, "claim_type": "factual_or_methodological_claim", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_url": "https://arxiv.org/abs/2402.03927", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope is appropriately bounded.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_with_scope", "publication_material": false }, { "claim_id": "A-005", "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Position bias observed; swap is diagnostic" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Study supports diagnostic, not bias-elimination guarantee.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-006", "cluster": "A", "article_id": "02", "article_file": "A/articles/02-public-benchmarks.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "GitHub issues and PRs, tested in repository environments" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweoriginal", "source_title": "SWE-bench", "source_url": "https://www.swebench.com/original.html", "source_location": "Overview", "support_or_contradiction": "Maintainer overview matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-007", "cluster": "A", "article_id": "02", "article_file": "A/articles/02-public-benchmarks.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Standardised scenarios and multiple metrics expose trade-offs" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Design description matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-008", "cluster": "A", "article_id": "02", "article_file": "A/articles/02-public-benchmarks.md", "claim_location": { "line": 37, "short_excerpt_or_precise_claim": "Benchmark exposure includes indirect routes" }, "claim_type": "factual_or_methodological_claim", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_url": "https://arxiv.org/abs/2402.03927", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Indirect exposure is discussed.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-009", "cluster": "A", "article_id": "03", "article_file": "A/articles/03-task-design.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "Guidance recommends typical, edge and adversarial cases" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "All three categories appear.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-010", "cluster": "A", "article_id": "03", "article_file": "A/articles/03-task-design.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "Oversampling requires separate reporting or credible weights" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Weighting interpretation follows aggregation definitions.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-011", "cluster": "A", "article_id": "03", "article_file": "A/articles/03-task-design.md", "claim_location": { "line": 57, "short_excerpt_or_precise_claim": "Questions sharing documents are not independent units" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Grouping follows dependence structure.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-012", "cluster": "A", "article_id": "04", "article_file": "A/articles/04-datasets-holdouts.md", "claim_location": { "line": 19, "short_excerpt_or_precise_claim": "Repeated tuning on a holdout makes it development feedback" }, "claim_type": "factual_or_methodological_claim", "source_id": "skleak", "source_title": "Common pitfalls and recommended practices", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_location": "Data leakage", "support_or_contradiction": "Test-based model selection violates holdout independence.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-013", "cluster": "A", "article_id": "04", "article_file": "A/articles/04-datasets-holdouts.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Closed-source contamination exposure and reproducibility limits" }, "claim_type": "factual_or_methodological_claim", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_url": "https://arxiv.org/abs/2402.03927", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Relevant limitations match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-014", "cluster": "A", "article_id": "04", "article_file": "A/articles/04-datasets-holdouts.md", "claim_location": { "line": 25, "short_excerpt_or_precise_claim": "Split related documents/conversations as groups" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Dependence requires group-aware treatment.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-015", "cluster": "A", "article_id": "05", "article_file": "A/articles/05-metrics-rubrics.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "Precision and recall have different denominators" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Definitions match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-016", "cluster": "A", "article_id": "05", "article_file": "A/articles/05-metrics-rubrics.md", "claim_location": { "line": 27, "short_excerpt_or_precise_claim": "Passing specified tests does not check every requirement" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweguide", "source_title": "Evaluation Guide", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_location": "Understanding Evaluation Results, count definitions", "support_or_contradiction": "Resolved is test-defined, not universal correctness.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-017", "cluster": "A", "article_id": "05", "article_file": "A/articles/05-metrics-rubrics.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "Metric, human and model evaluators; generic metrics can miss task" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Evaluator guidance matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-018", "cluster": "A", "article_id": "06", "article_file": "A/articles/06-human-model-judges.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Position bias varies across tested judges and tasks" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope matches the paper.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-019", "cluster": "A", "article_id": "06", "article_file": "A/articles/06-human-model-judges.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "Judges can prefer verbosity or familiar outputs" }, "claim_type": "factual_or_methodological_claim", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_url": "https://arxiv.org/abs/2306.05685", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Named biases match; no local rate claimed.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-020", "cluster": "A", "article_id": "06", "article_file": "A/articles/06-human-model-judges.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Overall agreement can conceal false passes in imbalanced labels" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Confusion-matrix reasoning is sound.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-021", "cluster": "A", "article_id": "07", "article_file": "A/articles/07-repeatability-uncertainty.md", "claim_location": { "line": 27, "short_excerpt_or_precise_claim": "Paired case differences reduce case-difficulty noise" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Pairing method matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-022", "cluster": "A", "article_id": "07", "article_file": "A/articles/07-repeatability-uncertainty.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "Paired bootstrap retains corresponding outcomes" }, "claim_type": "factual_or_methodological_claim", "source_id": "bootstrap", "source_title": "scipy.stats.bootstrap", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_location": "Introduction; paired parameter; warnings", "support_or_contradiction": "Shared-index resampling matches; grouping is an extension.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_with_scope", "publication_material": false }, { "claim_id": "A-023", "cluster": "A", "article_id": "07", "article_file": "A/articles/07-repeatability-uncertainty.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Normal approximations may fail for small n/few failures" }, "claim_type": "factual_or_methodological_claim", "source_id": "nistexact", "source_title": "Exact Binomial Confidence Limits", "source_url": "https://itl.nist.gov/div898/software/dataplot/refman2/auxillar/exacbici.htm", "source_location": "Description", "support_or_contradiction": "NIST explicitly warns of this.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-024", "cluster": "A", "article_id": "08", "article_file": "A/articles/08-fair-comparisons.md", "claim_location": { "line": 19, "short_excerpt_or_precise_claim": "Controlled and tailored systems answer different questions" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Common conditions support attribution; taxonomy is editorial.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-025", "cluster": "A", "article_id": "08", "article_file": "A/articles/08-fair-comparisons.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Extra attempts/resources change system interpretation" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Cost and system-boundary argument supports this.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-026", "cluster": "A", "article_id": "08", "article_file": "A/articles/08-fair-comparisons.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Paired cases and temporal interleaving avoid avoidable confounding" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Pairing supported; interleaving is reasonable experimental control.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-027", "cluster": "A", "article_id": "09", "article_file": "A/articles/09-latency-cost.md", "claim_location": { "line": 27, "short_excerpt_or_precise_claim": "Visible answer text may omit billed reasoning tokens" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Billing distinction matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-028", "cluster": "A", "article_id": "09", "article_file": "A/articles/09-latency-cost.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "2,000 input at $2/M and 400 output at $8/M cost $0.0072; 10,000 cost $72" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Recomputed: (4000+3200)/1e6=0.0072; times10000=72.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "A-029", "cluster": "A", "article_id": "09", "article_file": "A/articles/09-latency-cost.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "$72/8,000 accepted outcomes = $0.009" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Division verified; invented rates are clearly labelled.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "A-030", "cluster": "A", "article_id": "10", "article_file": "A/articles/10-agent-evaluation.md", "claim_location": { "line": 15, "short_excerpt_or_precise_claim": "Agents evaluated across interactive environments" }, "claim_type": "factual_or_methodological_claim", "source_id": "agentbench", "source_title": "AgentBench: Evaluating LLMs as Agents", "source_url": "https://arxiv.org/abs/2308.03688", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Benchmark description matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-031", "cluster": "A", "article_id": "10", "article_file": "A/articles/10-agent-evaluation.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "Final state and tool results can differ from success prose" }, "claim_type": "factual_or_methodological_claim", "source_id": "taumetric", "source_title": "\u03c4-bench, section 3", "source_url": "https://arxiv.org/html/2406.12045v1#S3", "source_location": "Section 3, reward and pass^k paragraphs", "support_or_contradiction": "State reward supports independent outcome checks.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-032", "cluster": "A", "article_id": "10", "article_file": "A/articles/10-agent-evaluation.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Errors differ from unresolved completed evaluations" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweguide", "source_title": "Evaluation Guide", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_location": "Understanding Evaluation Results, count definitions", "support_or_contradiction": "Count categories match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-033", "cluster": "A", "article_id": "11", "article_file": "A/articles/11-results-maintenance.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Several cases from one incident share dependence" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Grouping warning applies.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "A-034", "cluster": "A", "article_id": "11", "article_file": "A/articles/11-results-maintenance.md", "claim_location": { "line": 41, "short_excerpt_or_precise_claim": "Guidance recommends continuous evaluation and human calibration" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Both principles stated.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported", "publication_material": false }, { "claim_id": "A-035", "cluster": "A", "article_id": "11", "article_file": "A/articles/11-results-maintenance.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "Changing grader changes the measurement of identical output" }, "claim_type": "factual_or_methodological_claim", "source_id": "graders", "source_title": "Graders", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_location": "Types; best practices; grader hacking", "support_or_contradiction": "Separate instrument/version follows grader validation.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-28", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "B-001", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 22, "short_excerpt_or_precise_claim": "MMLU has 57 tasks, broad academic knowledge/problem solving" }, "claim_type": "factual_or_methodological_claim", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope and task count match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-002", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 22, "short_excerpt_or_precise_claim": "BIG-bench: 204 tasks, 450 authors, 132 institutions, expert raters" }, "claim_type": "factual_or_methodological_claim", "source_id": "bigbench", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "source_url": "https://arxiv.org/abs/2206.04615", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Counts and baseline match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-003", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 22, "short_excerpt_or_precise_claim": "HELM: seven metrics, 16 core/42 total scenarios" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "All counts match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-004", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 26, "short_excerpt_or_precise_claim": "Arena: >240,000 votes and crowd/expert agreement at paper date" }, "claim_type": "factual_or_methodological_claim", "source_id": "arena", "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "source_url": "https://arxiv.org/abs/2403.04132", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical report matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-005", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 26, "short_excerpt_or_precise_claim": "Private testing, selective disclosure and sampling asymmetry" }, "claim_type": "factual_or_methodological_claim", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "source_url": "https://arxiv.org/abs/2504.20879", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Findings faithfully attributed.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-006", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 26, "short_excerpt_or_precise_claim": "24 benchmarks and 46 lifecycle practices; replication/significance gaps" }, "claim_type": "factual_or_methodological_claim", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_url": "https://arxiv.org/abs/2411.12990", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Counts and findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-007", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 34, "short_excerpt_or_precise_claim": "Architectures introduce tool selection, data precision and handoffs" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Architecture distinctions match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-008", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "Original SWE-bench has 2,294 issues from 12 repositories" }, "claim_type": "factual_or_methodological_claim", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_url": "https://arxiv.org/abs/2310.06770", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Counts match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-009", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "Verified 500 and Bash Only common mini-SWE-agent" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "source_url": "https://www.swebench.com/", "source_location": "View descriptions", "support_or_contradiction": "Current maintainer descriptions match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-010", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "GAIA human 92% versus GPT-4 with plugins 15%" }, "claim_type": "factual_or_methodological_claim", "source_id": "gaia", "source_title": "GAIA: a benchmark for General AI Assistants", "source_url": "https://arxiv.org/abs/2311.12983", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical comparison matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-011", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "Final database state; <50% success and retail pass^8 <25%" }, "claim_type": "factual_or_methodological_claim", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original results and method match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-012", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 64, "short_excerpt_or_precise_claim": "8\u201318% HumanEval overlap; paraphrases evade n-gram filters" }, "claim_type": "factual_or_methodological_claim", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Paper reports both.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-013", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 72, "short_excerpt_or_precise_claim": "Accuracy-only agent evaluation and insufficient holdouts" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-014", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 22, "short_excerpt_or_precise_claim": "MMLU historical best models below experts on all 57 tasks" }, "claim_type": "factual_or_methodological_claim", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical scope retained.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-015", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 22, "short_excerpt_or_precise_claim": "BIG-bench scale/calibration and brittle breakthrough metrics" }, "claim_type": "factual_or_methodological_claim", "source_id": "bigbench", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "source_url": "https://arxiv.org/abs/2206.04615", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Summary matches findings.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-016", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 24, "short_excerpt_or_precise_claim": "7 metrics,16 core,26 targeted,30 models;17.9% to96.0% coverage" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Numerical associations match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-017", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 38, "short_excerpt_or_precise_claim": "Arena >240K votes and crowd/expert agreement" }, "claim_type": "factual_or_methodological_claim", "source_id": "arena", "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "source_url": "https://arxiv.org/abs/2403.04132", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical claims match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-018", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 40, "short_excerpt_or_precise_claim": "Data shares19.2%,20.4%,83 open models29.7%" }, "claim_type": "factual_or_methodological_claim", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "source_url": "https://arxiv.org/abs/2504.20879", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Figures and attribution match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-019", "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "claim_location": { "line": 42, "short_excerpt_or_precise_claim": "24 benchmarks,46 practices,significance and replication gaps" }, "claim_type": "factual_or_methodological_claim", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_url": "https://arxiv.org/abs/2411.12990", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Source matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-020", "cluster": "B", "article_id": "ai-eval-02", "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "claim_location": { "line": 26, "short_excerpt_or_precise_claim": "Typical/edge/adversarial dataset coverage" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Source lists these categories.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-021", "cluster": "B", "article_id": "ai-eval-02", "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "claim_location": { "line": 102, "short_excerpt_or_precise_claim": "Evaluation criteria drift as outputs are inspected" }, "claim_type": "factual_or_methodological_claim", "source_id": "shankar", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_url": "https://arxiv.org/abs/2404.12272", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Phenomenon correctly described.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-022", "cluster": "B", "article_id": "ai-eval-02", "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "claim_location": { "line": 114, "short_excerpt_or_precise_claim": "Power depends on effect and variance" }, "claim_type": "factual_or_methodological_claim", "source_id": "card", "source_title": "With Little Power Comes Great Responsibility", "source_url": "https://arxiv.org/abs/2010.06595", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Relevant methodological support.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-023", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 28, "short_excerpt_or_precise_claim": "Agent benchmarks can lack adequate holdouts" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-024", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "8\u201318% HumanEval overlap; contaminated13B matches larger models" }, "claim_type": "factual_or_methodological_claim", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Results match; not proof of any local model contamination.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-025", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 38, "short_excerpt_or_precise_claim": "TS-Guessing52%/57% missing MMLU options" }, "claim_type": "factual_or_methodological_claim", "source_id": "deng", "source_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "source_url": "https://arxiv.org/abs/2311.09783", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Values and task match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-026", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 40, "short_excerpt_or_precise_claim": "White/grey/black-box detection; update/rewrite/prevention" }, "claim_type": "factual_or_methodological_claim", "source_id": "cheng", "source_title": "A Survey on Data Contamination for Large Language Models", "source_url": "https://arxiv.org/abs/2502.14425", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Taxonomy matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-027", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 60, "short_excerpt_or_precise_claim": "Private testing/selective disclosure distort ranking" }, "claim_type": "factual_or_methodological_claim", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "source_url": "https://arxiv.org/abs/2504.20879", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Finding matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-028", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 30, "short_excerpt_or_precise_claim": "pass@k estimator1-C(n-c,k)/C(n,k)" }, "claim_type": "factual_or_methodological_claim", "source_id": "passk", "source_title": "Evaluating Large Language Models Trained on Code, section 2.1", "source_url": "https://arxiv.org/html/2107.03374v2#S2.SS1", "source_location": "Section 2.1, equation 1; Appendix A", "support_or_contradiction": "Correct for n>=k; add domain explicitly.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-029", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 32, "short_excerpt_or_precise_claim": "\u03c4-bench pass^k and <50% historical success" }, "claim_type": "factual_or_methodological_claim", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Definition and historical result match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-030", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 34, "short_excerpt_or_precise_claim": "SWE-bench test-based issue resolution" }, "claim_type": "factual_or_methodological_claim", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_url": "https://arxiv.org/abs/2310.06770", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-031", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 38, "short_excerpt_or_precise_claim": "Tool selection and data precision differ from functional correctness" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Named criteria match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-032", "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "claim_location": { "line": 38, "short_excerpt_or_precise_claim": "Criteria drift discovered through grading outputs" }, "claim_type": "factual_or_methodological_claim", "source_id": "shankar", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_url": "https://arxiv.org/abs/2404.12272", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Attribution matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-033", "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "claim_location": { "line": 72, "short_excerpt_or_precise_claim": ">80% human agreement; position/verbosity/self-enhancement bias" }, "claim_type": "factual_or_methodological_claim", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_url": "https://arxiv.org/abs/2306.05685", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical conditional finding matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-034", "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "claim_location": { "line": 76, "short_excerpt_or_precise_claim": "Pairwise or pass/fail recommended for judge reliability" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Recommendation is accurately attributed.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-035", "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "claim_location": { "line": 90, "short_excerpt_or_precise_claim": "Reasoning before scoring is recommended in guide" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Guide states it; local improvement remains conditional.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-036", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 28, "short_excerpt_or_precise_claim": "Seeds/training and simple format changes affect variance" }, "claim_type": "factual_or_methodological_claim", "source_id": "madaan", "source_title": "Quantifying Variance in Evaluation Benchmarks", "source_url": "https://arxiv.org/abs/2406.10229", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Study scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-037", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 34, "short_excerpt_or_precise_claim": "SE=sd/sqrt(n); BernoulliSE=sqrt(p(1-p)/n)" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Formulas match under stated independent-case assumptions.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-038", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 58, "short_excerpt_or_precise_claim": "Clustered standard errors up to roughly3x naive in real data" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Table4 maximum3.05 supports wording.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-039", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 66, "short_excerpt_or_precise_claim": "Repeat reductions one-third/one-half/limit two-thirds in example" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Section3.1 conditional example matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-040", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 68, "short_excerpt_or_precise_claim": "Temperature changes target behaviour; do not adjust solely for variance" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Section3.3 supports caution.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-041", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 74, "short_excerpt_or_precise_claim": "Unbiased pass@k formula" }, "claim_type": "factual_or_methodological_claim", "source_id": "passk", "source_title": "Evaluating Large Language Models Trained on Code, section 2.1", "source_url": "https://arxiv.org/html/2107.03374v2#S2.SS1", "source_location": "Section 2.1, equation 1; Appendix A", "support_or_contradiction": "Equation1 matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-042", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 76, "short_excerpt_or_precise_claim": "Retail pass^8 below25% with first-attempt below50%" }, "claim_type": "factual_or_methodological_claim", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Historical findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-043", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 82, "short_excerpt_or_precise_claim": "2,000-sentence MT tests approximately75% power for1BLEU" }, "claim_type": "factual_or_methodological_claim", "source_id": "card", "source_title": "With Little Power Comes Great Responsibility", "source_url": "https://arxiv.org/abs/2010.06595", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original abstract supports under assumptions.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-044", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 88, "short_excerpt_or_precise_claim": "969 independent questions for3point effect under given assumptions" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Equation9 worked value matches assumed difference variance1/9.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-045", "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "claim_location": { "line": 14, "short_excerpt_or_precise_claim": "Architectures have distinct sources of nondeterminism" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Official guide supports.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-046", "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "claim_location": { "line": 74, "short_excerpt_or_precise_claim": "Inspect has composable datasets/solvers/scorers/sandboxes" }, "claim_type": "factual_or_methodological_claim", "source_id": "inspect", "source_title": "Inspect", "source_url": "https://inspect.aisi.org.uk/", "source_location": "Overview, components and sandbox sections", "support_or_contradiction": "Maintainer framework matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-047", "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "claim_location": { "line": 100, "short_excerpt_or_precise_claim": "Bash Only holds common mini-SWE-agent environment" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "source_url": "https://www.swebench.com/", "source_location": "View descriptions", "support_or_contradiction": "View description matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-048", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 20, "short_excerpt_or_precise_claim": "Accuracy and cost should be jointly optimised" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original argument matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-049", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 36, "short_excerpt_or_precise_claim": "Half output may halve latency;half input may improve1\u20135%" }, "claim_type": "factual_or_methodological_claim", "source_id": "latency", "source_title": "Latency optimization", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_location": "Generate fewer tokens; fewer input tokens; make users wait less", "support_or_contradiction": "Documented heuristics; caveats retained.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-050", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "Reasoning tokens billed as output and occupy context" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Provider-specific statement matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-051", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 47, "short_excerpt_or_precise_claim": "Initial buffer of at least25,000 reasoning/output tokens" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Current guide recommendation matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-052", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 72, "short_excerpt_or_precise_claim": "Astra10/50,cache1,batch5/25 USDperM" }, "claim_type": "factual_or_methodological_claim", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Current standard/batch tables match; not an archived25Oct snapshot.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-053", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 73, "short_excerpt_or_precise_claim": "Luna0.10/0.50,cache0.01,batch0.05/0.25" }, "claim_type": "factual_or_methodological_claim", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Current standard/batch tables match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-054", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 74, "short_excerpt_or_precise_claim": "Fable5.1 input10/output50/cache read0.25" }, "claim_type": "factual_or_methodological_claim", "source_id": "claude", "source_title": "Plans and pricing", "source_url": "https://claude.com/pricing", "source_location": "Latest models, Haiku prompt-length tiers", "support_or_contradiction": "Current table matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-055", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 76, "short_excerpt_or_precise_claim": "Argon2/10/cache0.10,batch1/5 until30Nov;double1Dec" }, "claim_type": "factual_or_methodological_claim", "source_id": "gemini", "source_title": "Gemini Developer API pricing", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "source_location": "Gemini 4 Argon standard/batch tables", "support_or_contradiction": "Current dated schedule matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-056", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 78, "short_excerpt_or_precise_claim": "Anthropic batch discount50%" }, "claim_type": "factual_or_methodological_claim", "source_id": "claudedetail", "source_title": "Pricing", "source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "source_location": "Model, batch and long-context pricing", "support_or_contradiction": "Official batch table confirms.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-057", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "System/model needs conflated;holdouts and standardisation weak" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-058", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 37, "short_excerpt_or_precise_claim": "Original best SWE-bench model1.96%;2,294issues/12repos" }, "claim_type": "factual_or_methodological_claim", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_url": "https://arxiv.org/abs/2310.06770", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original historical result matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-059", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 37, "short_excerpt_or_precise_claim": "Verified500,Lite300,Multilingual300/42repos/9languages" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "source_url": "https://www.swebench.com/", "source_location": "View descriptions", "support_or_contradiction": "All current maintainer counts match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-060", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Policy/API conversations;database state and pass^k;historical rates" }, "claim_type": "factual_or_methodological_claim", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Method and reported outcomes match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-061", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 41, "short_excerpt_or_precise_claim": "466questions,92%humans,15%GPT4,300answers retained" }, "claim_type": "factual_or_methodological_claim", "source_id": "gaia", "source_title": "GAIA: a benchmark for General AI Assistants", "source_url": "https://arxiv.org/abs/2311.12983", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "All figures match original paper.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-062", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "Continuous newer contest problems mitigate exposure risk" }, "claim_type": "factual_or_methodological_claim", "source_id": "livecode", "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "source_url": "https://arxiv.org/abs/2403.07974", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Design supported; cutoff knowledge remains a limitation.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-063", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 73, "short_excerpt_or_precise_claim": "Start debugging with traces,then datasets for repeatability" }, "claim_type": "factual_or_methodological_claim", "source_id": "agentevals", "source_title": "Evaluate agent workflows", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_location": "Start with traces; datasets and repeatable evals", "support_or_contradiction": "Workflow matches official guide.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-064", "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "claim_location": { "line": 85, "short_excerpt_or_precise_claim": "Composable agent/tool/scorer/sandbox evaluation" }, "claim_type": "factual_or_methodological_claim", "source_id": "inspect", "source_title": "Inspect", "source_url": "https://inspect.aisi.org.uk/", "source_location": "Overview, components and sandbox sections", "support_or_contradiction": "Maintainer documentation matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-065", "cluster": "B", "article_id": "ai-eval-10", "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "Most reviewed benchmarks lack significance/replication reporting" }, "claim_type": "factual_or_methodological_claim", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_url": "https://arxiv.org/abs/2411.12990", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Finding matches; scope is reviewed benchmarks.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-066", "cluster": "B", "article_id": "ai-eval-10", "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "claim_location": { "line": 75, "short_excerpt_or_precise_claim": "Selection and data access can distort public ranking" }, "claim_type": "factual_or_methodological_claim", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "source_url": "https://arxiv.org/abs/2504.20879", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Study supports analogy; company application is inference.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-067", "cluster": "B", "article_id": "ai-eval-10", "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "claim_location": { "line": 112, "short_excerpt_or_precise_claim": "Evals read-only31October2026;shutdown30November2026" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Current notice matches dated schedule.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported", "publication_material": false }, { "claim_id": "B-068", "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "claim_location": { "line": 90, "short_excerpt_or_precise_claim": "Hypothetical78%vs74%,n120,SE3.9pp,CI-3.6to11.6;power about900" }, "claim_type": "hypothetical_calculation", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Using Bernoulli variances and rho0.5 givesSE0.03896,n\u2248894; rounding works.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "B-069", "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "claim_location": { "line": 50, "short_excerpt_or_precise_claim": "Hypothetical paired example SE/interval and900task power" }, "claim_type": "hypothetical_calculation", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Independent arithmetic gives3.896pp,[-3.64,11.64],n\u2248894.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "B-070", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 44, "short_excerpt_or_precise_claim": "Initial release400problems May2023\u2013May2024" }, "claim_type": "factual_or_methodological_claim", "source_id": "livecodev1", "source_title": "LiveCodeBench, initial version", "source_url": "https://arxiv.org/abs/2403.07974v1", "source_location": "v1 abstract, submitted 12 March 2024", "support_or_contradiction": "Initial v1 ends February2024; May2024 belongs to revised v2.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "inaccurate_version_label", "publication_material": false }, { "claim_id": "B-071", "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "claim_location": { "line": 88, "short_excerpt_or_precise_claim": "Unknown provider decontamination makes score an upper bound" }, "claim_type": "factual_or_methodological_claim", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "No guaranteed bound follows from missing disclosure.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "unsupported_generalisation", "publication_material": true }, { "claim_id": "B-072", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 44, "short_excerpt_or_precise_claim": "Token F1 rewards verbosity" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Extra unmatched tokens lower precision/F1; overbroad wording.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "inaccurate_generalisation", "publication_material": false }, { "claim_id": "B-073", "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "claim_location": { "line": 74, "short_excerpt_or_precise_claim": "Pooling tasks answers performance across traffic" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Only if sampled frequencies represent traffic; qualification missing.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "missing_condition", "publication_material": false }, { "claim_id": "B-074", "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "claim_location": { "line": 82, "short_excerpt_or_precise_claim": "Order swapping and averaging cancels position effect" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Symmetric pure first-position preference cancels; general bias may persist.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "supported_only_in_stated_special_case", "publication_material": false }, { "claim_id": "B-075", "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "claim_location": { "line": 110, "short_excerpt_or_precise_claim": "Higher effort always increases tokens and latency" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Effort guides adaptive computation; no per-request monotonic guarantee.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "unsupported_absolute", "publication_material": false }, { "claim_id": "B-076", "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "claim_location": { "line": 114, "short_excerpt_or_precise_claim": "Same nominal effort provides matched effort across models" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Model-dependent labels do not establish matched compute.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "misleading_equivalence", "publication_material": true }, { "claim_id": "B-077", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 75, "short_excerpt_or_precise_claim": "Haiku5.5 input0.10/output0.50 without prompt-length qualification" }, "claim_type": "factual_or_methodological_claim", "source_id": "claude", "source_title": "Plans and pricing", "source_url": "https://claude.com/pricing", "source_location": "Latest models, Haiku prompt-length tiers", "support_or_contradiction": "Correct only at<=100K; above threshold0.50/2.50, fivefold higher.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "material_missing_price_condition", "publication_material": true }, { "claim_id": "B-078", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 53, "short_excerpt_or_precise_claim": "Formula uses output after defining output as visible answer" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Formula must use total billed output including reasoning, or add it once.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "material_accounting_ambiguity", "publication_material": true }, { "claim_id": "B-079", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 88, "short_excerpt_or_precise_claim": "15,000input/2,000output,1.1attempts,85%success cost0.32/0.0032" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "0.25*1.1/.85=0.323529;0.0025*1.1/.85=0.003235.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "B-080", "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "claim_location": { "line": 102, "short_excerpt_or_precise_claim": "Incremental success cost about1.80;cache case about0.24" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "(27.5-.275)/15=1.815;(.0825+.1)*1.1/.85=.23618. Assumes billed tokens and cache hits.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-25", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-001", "cluster": "C", "article_id": "P01", "article_file": "C/articles/ai-model-benchmarking-guide.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "MMLU covers57subjects including mathematics/history/law" }, "claim_type": "factual_or_methodological_claim", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Task scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-002", "cluster": "C", "article_id": "P01", "article_file": "C/articles/ai-model-benchmarking-guide.md", "claim_location": { "line": 73, "short_excerpt_or_precise_claim": "Task-specific evaluation and human-calibrated scoring" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Both process principles match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-003", "cluster": "C", "article_id": "P01", "article_file": "C/articles/ai-model-benchmarking-guide.md", "claim_location": { "line": 91, "short_excerpt_or_precise_claim": "Repeated outputs are not independent new tasks" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Nesting warning matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-004", "cluster": "C", "article_id": "S01", "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "HumanEval generates programs from docstrings and tests correctness" }, "claim_type": "factual_or_methodological_claim", "source_id": "humaneval", "source_title": "Evaluating Large Language Models Trained on Code", "source_url": "https://arxiv.org/abs/2107.03374", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Task description matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-005", "cluster": "C", "article_id": "S01", "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "At-least-one success among attempts differs from first attempt" }, "claim_type": "factual_or_methodological_claim", "source_id": "passk", "source_title": "Evaluating Large Language Models Trained on Code, section 2.1", "source_url": "https://arxiv.org/html/2107.03374v2#S2.SS1", "source_location": "Section 2.1, equation 1; Appendix A", "support_or_contradiction": "Metric interpretation matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-006", "cluster": "C", "article_id": "S01", "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "Default Verified Bash Only uses common mini-SWE-agent" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "source_url": "https://www.swebench.com/", "source_location": "View descriptions", "support_or_contradiction": "Current description matches22Oct claim; historical page not archived.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-007", "cluster": "C", "article_id": "S01", "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "claim_location": { "line": 53, "short_excerpt_or_precise_claim": "Multi-scenario,multi-metric evaluation includes robustness/efficiency" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Design matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-008", "cluster": "C", "article_id": "S02", "article_file": "C/articles/design-llm-evaluation-tasks.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "Providing later conversation outcomes leaks future information" }, "claim_type": "factual_or_methodological_claim", "source_id": "skleak", "source_title": "Common pitfalls and recommended practices", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_location": "Data leakage", "support_or_contradiction": "Prediction-time information principle applies.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "C-009", "cluster": "C", "article_id": "S02", "article_file": "C/articles/design-llm-evaluation-tasks.md", "claim_location": { "line": 55, "short_excerpt_or_precise_claim": "CheckList organises capabilities and test types" }, "claim_type": "factual_or_methodological_claim", "source_id": "checklist", "source_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList", "source_url": "https://aclanthology.org/2020.acl-main.442/", "source_location": "Abstract and bibliographic record", "support_or_contradiction": "Matrix described in original abstract.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-010", "cluster": "C", "article_id": "S02", "article_file": "C/articles/design-llm-evaluation-tasks.md", "claim_location": { "line": 57, "short_excerpt_or_precise_claim": "Challenge-heavy sample does not estimate ordinary traffic without weighting" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Aggregation interpretation is sound.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "C-011", "cluster": "C", "article_id": "S03", "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Leakage uses information unavailable at prediction time;split before fit" }, "claim_type": "factual_or_methodological_claim", "source_id": "skleak", "source_title": "Common pitfalls and recommended practices", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_location": "Data leakage", "support_or_contradiction": "Definition and ordering match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-012", "cluster": "C", "article_id": "S03", "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "Detection research studies access without disclosed training corpus" }, "claim_type": "factual_or_methodological_claim", "source_id": "shi", "source_title": "Detecting Pretraining Data from Large Language Models", "source_url": "https://arxiv.org/abs/2310.16789", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Research question accurately bounded.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-013", "cluster": "C", "article_id": "S03", "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Dataset records document motivation,composition,collection,use" }, "claim_type": "factual_or_methodological_claim", "source_id": "datasheets", "source_title": "Datasheets for Datasets", "source_url": "https://arxiv.org/abs/1803.09010", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Documentation precedent matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-014", "cluster": "C", "article_id": "S03", "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "claim_location": { "line": 65, "short_excerpt_or_precise_claim": "Canonical vs shuffled likelihood relies on exchangeability" }, "claim_type": "factual_or_methodological_claim", "source_id": "oren", "source_title": "Proving Test Set Contamination in Black Box Language Models", "source_url": "https://arxiv.org/abs/2310.17623", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Method and assumption match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-015", "cluster": "C", "article_id": "S04", "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "Missing/empty/unscorable differ from unresolved outcomes" }, "claim_type": "factual_or_methodological_claim", "source_id": "sweguide", "source_title": "Evaluation Guide", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_location": "Understanding Evaluation Results, count definitions", "support_or_contradiction": "Methodological distinction supported by maintainer counts.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-016", "cluster": "C", "article_id": "S04", "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "claim_location": { "line": 47, "short_excerpt_or_precise_claim": "Macro averaging weights classes equally" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Definition matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-017", "cluster": "C", "article_id": "S05", "article_file": "C/articles/human-reviewers-llm-judges.md", "claim_location": { "line": 49, "short_excerpt_or_precise_claim": "Judge position,verbosity,self-enhancement and reasoning limits" }, "claim_type": "factual_or_methodological_claim", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_url": "https://arxiv.org/abs/2306.05685", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Names and scope match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-018", "cluster": "C", "article_id": "S05", "article_file": "C/articles/human-reviewers-llm-judges.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Order-sensitive verdicts need investigation" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Diagnostic follows observed bias; no guarantee claimed.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "C-019", "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "Ten repeats do not create ten independent conversations" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Independent-unit principle matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-020", "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "95%frequentist interval is long-run procedural coverage" }, "claim_type": "factual_or_methodological_claim", "source_id": "bootstrap", "source_title": "scipy.stats.bootstrap", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_location": "Introduction; paired parameter; warnings", "support_or_contradiction": "Coverage interpretation matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-021", "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "claim_location": { "line": 49, "short_excerpt_or_precise_claim": "Paired bootstrap uses shared indices with replacement" }, "claim_type": "factual_or_methodological_claim", "source_id": "bootstrap", "source_title": "scipy.stats.bootstrap", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_location": "Introduction; paired parameter; warnings", "support_or_contradiction": "Procedure matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-022", "cluster": "C", "article_id": "S07", "article_file": "C/articles/fair-ai-model-comparisons.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "Meaning-preserving format sensitivity;weak transfer across models" }, "claim_type": "factual_or_methodological_claim", "source_id": "sclar", "source_title": "Quantifying Language Models\u2019 Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting", "source_url": "https://arxiv.org/abs/2310.11324", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope and findings match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-023", "cluster": "C", "article_id": "S07", "article_file": "C/articles/fair-ai-model-comparisons.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Reasoning tokens use context and billed output" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Provider-specific fact matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-024", "cluster": "C", "article_id": "S07", "article_file": "C/articles/fair-ai-model-comparisons.md", "claim_location": { "line": 75, "short_excerpt_or_precise_claim": "EleutherAI provides shared language-model evaluation framework" }, "claim_type": "factual_or_methodological_claim", "source_id": "harness", "source_title": "Language Model Evaluation Harness", "source_url": "https://github.com/EleutherAI/lm-evaluation-harness", "source_location": "README Overview", "support_or_contradiction": "Repository overview matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-025", "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Streaming affects visible waiting,not identical completion boundary" }, "claim_type": "factual_or_methodological_claim", "source_id": "latency", "source_title": "Latency optimization", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_location": "Generate fewer tokens; fewer input tokens; make users wait less", "support_or_contradiction": "Guide supports visible-wait distinction.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-026", "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "claim_location": { "line": 41, "short_excerpt_or_precise_claim": "Pricing separates input/output/cached input/tool categories" }, "claim_type": "factual_or_methodological_claim", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Current category structure matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-027", "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Rendered prefixes and model-dependent eligibility matter" }, "claim_type": "factual_or_methodological_claim", "source_id": "caching", "source_title": "Prompt caching", "source_url": "https://developers.openai.com/api/docs/guides/prompt-caching", "source_location": "Rendered-prefix match, lookup boundaries, gotchas", "support_or_contradiction": "Current cache rules support caution.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-028", "cluster": "C", "article_id": "S09", "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "\u03c4-bench uses annotated final state,policies,simulated users" }, "claim_type": "factual_or_methodological_claim", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_url": "https://arxiv.org/abs/2406.12045", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original design matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-029", "cluster": "C", "article_id": "S09", "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "Final state alone cannot prove permission adherence" }, "claim_type": "factual_or_methodological_claim", "source_id": "taumetric", "source_title": "\u03c4-bench, section 3", "source_url": "https://arxiv.org/html/2406.12045v1#S3", "source_location": "Section 3, reward and pass^k paragraphs", "support_or_contradiction": "Paper itself notes confirmation violation despite passing state.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-030", "cluster": "C", "article_id": "S09", "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "claim_location": { "line": 73, "short_excerpt_or_precise_claim": "pass^k all independent trials;pass@k at least one" }, "claim_type": "factual_or_methodological_claim", "source_id": "taumetric", "source_title": "\u03c4-bench, section 3", "source_url": "https://arxiv.org/html/2406.12045v1#S3", "source_location": "Section 3, reward and pass^k paragraphs", "support_or_contradiction": "Notation and task averaging correct.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-031", "cluster": "C", "article_id": "S09", "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "claim_location": { "line": 75, "short_excerpt_or_precise_claim": "Raising aggregate mean success to k is not task-averaged reliability" }, "claim_type": "factual_or_methodological_claim", "source_id": "taumetric", "source_title": "\u03c4-bench, section 3", "source_url": "https://arxiv.org/html/2406.12045v1#S3", "source_location": "Section 3, reward and pass^k paragraphs", "support_or_contradiction": "Nonlinearity and task-level estimator support warning.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-032", "cluster": "C", "article_id": "S10", "article_file": "C/articles/maintain-llm-evaluation-suite.md", "claim_location": { "line": 23, "short_excerpt_or_precise_claim": "Continuous evaluation and adding monitored cases" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Guide recommends both.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "C-033", "cluster": "C", "article_id": "S10", "article_file": "C/articles/maintain-llm-evaluation-suite.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Different grades on identical output may be grader change" }, "claim_type": "factual_or_methodological_claim", "source_id": "graders", "source_title": "Graders", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_location": "Types; best practices; grader hacking", "support_or_contradiction": "Instrument distinction follows grader validation.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "C-034", "cluster": "C", "article_id": "S10", "article_file": "C/articles/maintain-llm-evaluation-suite.md", "claim_location": { "line": 63, "short_excerpt_or_precise_claim": "Incident-only samples cannot estimate all-failure prevalence" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Selection/weighting reasoning is sound.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "C-035", "cluster": "C", "article_id": "P01", "article_file": "C/articles/ai-model-benchmarking-guide.md", "claim_location": { "line": 105, "short_excerpt_or_precise_claim": "Hypothetical$12/80accepted=$0.15" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Recomputed12/80=0.15; not a measurement.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-036", "cluster": "C", "article_id": "S04", "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "100-case example TP15,FP10,FN5,TN70;precision60%,recall75%,F1~66.7%" }, "claim_type": "hypothetical_calculation", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "15/25=.6;15/20=.75;2*.6*.75/1.35=.666667.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-037", "cluster": "C", "article_id": "S04", "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "Accuracy85%;always-negative baseline80%" }, "claim_type": "hypothetical_calculation", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "(15+70)/100=.85;(10+70)/100=.8.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-038", "cluster": "C", "article_id": "S05", "article_file": "C/articles/human-reviewers-llm-judges.md", "claim_location": { "line": 61, "short_excerpt_or_precise_claim": "Always-pass judge90%agreement detects zero failures" }, "claim_type": "hypothetical_calculation", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "90matching passes/100;0/10failures detected.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-039", "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Paired75both pass,10both fail,5Aonly,10Bonly =>80/85 and15discordant" }, "claim_type": "hypothetical_calculation", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Counts sum100;A80,B85,difference5pp,discordance15.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-040", "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "80/100Wilson95%interval71.1\u201386.7%" }, "claim_type": "hypothetical_calculation", "source_id": "nist", "source_title": "7.2.4.1. Confidence intervals", "source_url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm", "source_location": "Wilson formulas", "support_or_contradiction": "Usingz1.96 gives[.7111690381,.8666340774].", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-041", "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "claim_location": { "line": 59, "short_excerpt_or_precise_claim": "2,000at$2/M plus500at$8/M=$0.008" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "2000*2/1e6+500*8/1e6=.008.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "C-042", "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "claim_location": { "line": 61, "short_excerpt_or_precise_claim": "Run cost/accepted=$0.015and~$0.0167" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "1.20/80=.015;1.50/90=.0166667.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "D-001", "cluster": "D", "article_id": "model-benchmarking", "article_file": "D/articles/choosing-model-benchmark.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "MMLU57subjects,broad academic/professional task" }, "claim_type": "factual_or_methodological_claim", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Scope and historical citation year match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-002", "cluster": "D", "article_id": "model-benchmarking", "article_file": "D/articles/choosing-model-benchmark.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "HELM multiple scenarios and seven named desiderata" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "All named dimensions present.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-003", "cluster": "D", "article_id": "model-benchmarking", "article_file": "D/articles/choosing-model-benchmark.md", "claim_location": { "line": 76, "short_excerpt_or_precise_claim": "Official pricing includes token categories and tool charges" }, "claim_type": "factual_or_methodological_claim", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Category claim matches current page.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-004", "cluster": "D", "article_id": "model-benchmarking", "article_file": "D/articles/choosing-model-benchmark.md", "claim_location": { "line": 84, "short_excerpt_or_precise_claim": "SWE-bench uses real issues and codebase changes" }, "claim_type": "factual_or_methodological_claim", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_url": "https://arxiv.org/abs/2310.06770", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original benchmark scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-005", "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "57-subject multiple-choice scope" }, "claim_type": "factual_or_methodological_claim", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "source_url": "https://arxiv.org/abs/2009.03300", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Task framing matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-006", "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Scenario and multi-metric coverage exposes tradeoffs" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Original design matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-007", "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Position bias can change preferences after order swap" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Substance supported; author label separately corrected.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-008", "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "Rephrased samples reveal limits of exposure checks" }, "claim_type": "factual_or_methodological_claim", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Paper supports paraphrase-sensitive contamination concern.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-009", "cluster": "D", "article_id": "task-evaluation", "article_file": "D/articles/build-task-eval.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Synthetic/challenge cases add coverage without prevalence evidence" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Dataset types support design; prevalence inference needs traffic.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-010", "cluster": "D", "article_id": "task-evaluation", "article_file": "D/articles/build-task-eval.md", "claim_location": { "line": 47, "short_excerpt_or_precise_claim": "Incident/template-related examples share dependence" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Grouping follows independent-unit principle.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-011", "cluster": "D", "article_id": "task-evaluation", "article_file": "D/articles/build-task-eval.md", "claim_location": { "line": 49, "short_excerpt_or_precise_claim": "Tuning on holdout failures erodes independent test status" }, "claim_type": "factual_or_methodological_claim", "source_id": "skleak", "source_title": "Common pitfalls and recommended practices", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_location": "Data leakage", "support_or_contradiction": "Test-based selection is leakage.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-012", "cluster": "D", "article_id": "dataset-integrity", "article_file": "D/articles/datasets-holdouts-contamination.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Shared source/customer rows may need grouped analysis" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Dependence structure supports grouping.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-013", "cluster": "D", "article_id": "dataset-integrity", "article_file": "D/articles/datasets-holdouts-contamination.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Paraphrase variants can bypass exact-string exposure checks" }, "claim_type": "factual_or_methodological_claim", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_url": "https://arxiv.org/abs/2311.04850", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Core finding supports stated limitation.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-014", "cluster": "D", "article_id": "dataset-integrity", "article_file": "D/articles/datasets-holdouts-contamination.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "2026study separates inflated absolute scores from relative order" }, "claim_type": "factual_or_methodological_claim", "source_id": "xiao", "source_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards", "source_url": "https://arxiv.org/abs/2609.02899", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Real paper; cautious study-specific paraphrase supported.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-015", "cluster": "D", "article_id": "evaluation-metrics", "article_file": "D/articles/metrics-rubrics.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Precision vs recall;95%routine baseline can miss all urgent cases" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Definitions and illustrative imbalance arithmetic match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-016", "cluster": "D", "article_id": "evaluation-metrics", "article_file": "D/articles/metrics-rubrics.md", "claim_location": { "line": 31, "short_excerpt_or_precise_claim": "Generic academic metrics alone may miss intended task" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Guidance matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-017", "cluster": "D", "article_id": "evaluation-metrics", "article_file": "D/articles/metrics-rubrics.md", "claim_location": { "line": 45, "short_excerpt_or_precise_claim": "String,similarity,code/model graders;test known quality examples" }, "claim_type": "factual_or_methodological_claim", "source_id": "graders", "source_title": "Graders", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_location": "Types; best practices; grader hacking", "support_or_contradiction": "Types and validation method match.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-018", "cluster": "D", "article_id": "evaluation-metrics", "article_file": "D/articles/metrics-rubrics.md", "claim_location": { "line": 53, "short_excerpt_or_precise_claim": "Macro equally weights classes;prevalence-weighted answers distribution" }, "claim_type": "factual_or_methodological_claim", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_location": "Definitions; average parameter", "support_or_contradiction": "Definitions match; representativeness condition retained.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-019", "cluster": "D", "article_id": "human-judging", "article_file": "D/articles/human-evaluation-model-judges.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Position bias observed in judge research" }, "claim_type": "factual_or_methodological_claim", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Real supporting paper; incorrect author label is distinct.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-020", "cluster": "D", "article_id": "human-judging", "article_file": "D/articles/human-evaluation-model-judges.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Model judges can favour verbosity or own-style outputs" }, "claim_type": "factual_or_methodological_claim", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_url": "https://arxiv.org/abs/2306.05685", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Named bias mechanisms supported by original study.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-021", "cluster": "D", "article_id": "human-judging", "article_file": "D/articles/human-evaluation-model-judges.md", "claim_location": { "line": 51, "short_excerpt_or_precise_claim": "Specific criteria,examples,human calibration recommended" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Guidance matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-022", "cluster": "D", "article_id": "repeatability", "article_file": "D/articles/repeatability-uncertainty.md", "claim_location": { "line": 30, "short_excerpt_or_precise_claim": "100cases*3outputs is100cases,not300independent cases" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Nesting warning matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-023", "cluster": "D", "article_id": "repeatability", "article_file": "D/articles/repeatability-uncertainty.md", "claim_location": { "line": 42, "short_excerpt_or_precise_claim": "Resample independent groups rather than correlated turns" }, "claim_type": "factual_or_methodological_claim", "source_id": "bootstrap", "source_title": "scipy.stats.bootstrap", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_location": "Introduction; paired parameter; warnings", "support_or_contradiction": "Paired/group extension appropriate with target weighting stated.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-024", "cluster": "D", "article_id": "repeatability", "article_file": "D/articles/repeatability-uncertainty.md", "claim_location": { "line": 44, "short_excerpt_or_precise_claim": "Benchmark variance affects comparisons and uncertainty" }, "claim_type": "factual_or_methodological_claim", "source_id": "bouthillier", "source_title": "Accounting for Variance in Machine Learning Benchmarks", "source_url": "https://arxiv.org/abs/2103.03098", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "General finding supported; local bootstrap is editorial method.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-025", "cluster": "D", "article_id": "fair-comparison", "article_file": "D/articles/fair-model-comparisons.md", "claim_location": { "line": 19, "short_excerpt_or_precise_claim": "Controlled substitution and deployable comparison differ in attribution" }, "claim_type": "factual_or_methodological_claim", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "source_url": "https://arxiv.org/abs/2211.09110", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Common-condition principle supports distinction.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-026", "cluster": "D", "article_id": "fair-comparison", "article_file": "D/articles/fair-model-comparisons.md", "claim_location": { "line": 21, "short_excerpt_or_precise_claim": "Model APIs expose model-dependent control settings" }, "claim_type": "factual_or_methodological_claim", "source_id": "reasoning", "source_title": "Reasoning models", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_location": "Reasoning effort; managing reasoning tokens", "support_or_contradiction": "Current effort documentation supports asymmetry.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-027", "cluster": "D", "article_id": "fair-comparison", "article_file": "D/articles/fair-model-comparisons.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Paired inputs reduce case-mix noise" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Pairing matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-028", "cluster": "D", "article_id": "latency-cost", "article_file": "D/articles/latency-cost.md", "claim_location": { "line": 29, "short_excerpt_or_precise_claim": "Token categories and tool charges differ in billing" }, "claim_type": "factual_or_methodological_claim", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Current official structure matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-029", "cluster": "D", "article_id": "latency-cost", "article_file": "D/articles/latency-cost.md", "claim_location": { "line": 37, "short_excerpt_or_precise_claim": "First visible streaming output differs from useful complete result" }, "claim_type": "factual_or_methodological_claim", "source_id": "latency", "source_title": "Latency optimization", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_location": "Generate fewer tokens; fewer input tokens; make users wait less", "support_or_contradiction": "Visible-wait distinction is sound.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-030", "cluster": "D", "article_id": "agent-evaluation", "article_file": "D/articles/agent-system-evaluation.md", "claim_location": { "line": 33, "short_excerpt_or_precise_claim": "Trace grading can assess tools,handoffs,workflow behaviour" }, "claim_type": "factual_or_methodological_claim", "source_id": "agentevals", "source_title": "Evaluate agent workflows", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_location": "Start with traces; datasets and repeatable evals", "support_or_contradiction": "Official workflow scope matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-031", "cluster": "D", "article_id": "agent-evaluation", "article_file": "D/articles/agent-system-evaluation.md", "claim_location": { "line": 43, "short_excerpt_or_precise_claim": "2026SWE-Bench Pro Verified paper reports leakage and task-quality concerns" }, "claim_type": "factual_or_methodological_claim", "source_id": "pro", "source_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents", "source_url": "https://arxiv.org/abs/2609.08149", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Real preprint; specific-version caveat appropriate.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-032", "cluster": "D", "article_id": "agent-evaluation", "article_file": "D/articles/agent-system-evaluation.md", "claim_location": { "line": 61, "short_excerpt_or_precise_claim": "Changing scaffold confounds attribution to model" }, "claim_type": "factual_or_methodological_claim", "source_id": "kapoor", "source_title": "AI Agents That Matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "System/model needs distinction supports ablation rationale.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-033", "cluster": "D", "article_id": "evaluation-maintenance", "article_file": "D/articles/maintaining-evaluations.md", "claim_location": { "line": 25, "short_excerpt_or_precise_claim": "Continuous checks and expanding suite from failures recommended" }, "claim_type": "factual_or_methodological_claim", "source_id": "eval", "source_title": "Evaluation best practices", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_location": "Evals tips; design; human/model grading; architecture; transition notice", "support_or_contradiction": "Guidance matches.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-034", "cluster": "D", "article_id": "evaluation-maintenance", "article_file": "D/articles/maintaining-evaluations.md", "claim_location": { "line": 38, "short_excerpt_or_precise_claim": "New cases may change score independently of model" }, "claim_type": "factual_or_methodological_claim", "source_id": "millerfull", "source_title": "Adding Error Bars to Evals, full text", "source_url": "https://arxiv.org/html/2411.00640v1", "source_location": "Sections 2\u20135; table 4; equations 1\u20139", "support_or_contradiction": "Population/sample distinction supports bridge comparison.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported_inference", "publication_material": false }, { "claim_id": "D-035", "cluster": "D", "article_id": "evaluation-maintenance", "article_file": "D/articles/maintaining-evaluations.md", "claim_location": { "line": 46, "short_excerpt_or_precise_claim": "Judge change can move ratings for same output" }, "claim_type": "factual_or_methodological_claim", "source_id": "graders", "source_title": "Graders", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_location": "Types; best practices; grader hacking", "support_or_contradiction": "Instrument validation supports version separation.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "supported", "publication_material": false }, { "claim_id": "D-036", "cluster": "D", "article_id": "latency-cost", "article_file": "D/articles/latency-cost.md", "claim_location": { "line": 57, "short_excerpt_or_precise_claim": "HypotheticalA$18+60*4*$1=$258;B$27+25*4*$1=$127" }, "claim_type": "hypothetical_calculation", "source_id": "pricing", "source_title": "Pricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_location": "Token tables and tool charges", "support_or_contradiction": "Both totals recomputed; corrected-success denominator left explicit.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "arithmetic_verified", "publication_material": false }, { "claim_id": "D-037", "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "claim_location": { "line": 39, "short_excerpt_or_precise_claim": "Zheng et al.,2024 associated with2406.07791" }, "claim_type": "bibliographic_association", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Lead author is Lin Shi; first submission2024,latest version2025. Real source supports claim.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "inaccurate_bibliographic_author", "publication_material": false }, { "claim_id": "D-038", "cluster": "D", "article_id": "human-judging", "article_file": "D/articles/human-evaluation-model-judges.md", "claim_location": { "line": 35, "short_excerpt_or_precise_claim": "Zheng et al.,2024 associated with2406.07791" }, "claim_type": "bibliographic_association", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_url": "https://arxiv.org/abs/2406.07791", "source_location": "Title, authors, submission history and abstract", "support_or_contradiction": "Same wrong author association; real title/URL and supported finding.", "retrieval_date": "2026-10-28", "candidate_research_date": "2026-10-22", "verdict": "inaccurate_bibliographic_author", "publication_material": false } ], "substantive_claim_count": 193, "total_claim_record_count": 195, "per_article_substantive_counts": { "A/articles/01-benchmarking-models.md": 5, "A/articles/02-public-benchmarks.md": 3, "A/articles/03-task-design.md": 3, "A/articles/04-datasets-holdouts.md": 3, "A/articles/05-metrics-rubrics.md": 3, "A/articles/06-human-model-judges.md": 3, "A/articles/07-repeatability-uncertainty.md": 3, "A/articles/08-fair-comparisons.md": 3, "A/articles/09-latency-cost.md": 3, "A/articles/10-agent-evaluation.md": 3, "A/articles/11-results-maintenance.md": 3, "B/articles/evaluating-ai-models-and-agent-systems.md": 14, "B/articles/public-benchmarks-vs-workload-evaluation.md": 6, "B/articles/designing-evaluation-tasks-and-datasets.md": 3, "B/articles/contamination-leakage-and-holdouts.md": 7, "B/articles/objective-metrics-and-rubrics.md": 6, "B/articles/human-evaluation-and-model-judges.md": 5, "B/articles/repeatability-variance-and-uncertainty.md": 10, "B/articles/prompt-tool-and-configuration-fairness.md": 5, "B/articles/latency-token-usage-and-cost.md": 13, "B/articles/evaluating-agents-and-tool-use.md": 8, "B/articles/interpreting-results-and-maintaining-evals.md": 3, "C/articles/ai-model-benchmarking-guide.md": 4, "C/articles/read-ai-benchmark-leaderboards.md": 4, "C/articles/design-llm-evaluation-tasks.md": 3, "C/articles/evaluation-dataset-leakage-holdouts.md": 4, "C/articles/llm-evaluation-metrics-rubrics.md": 4, "C/articles/human-reviewers-llm-judges.md": 3, "C/articles/benchmark-uncertainty-repeatability.md": 5, "C/articles/fair-ai-model-comparisons.md": 3, "C/articles/ai-latency-cost-per-successful-task.md": 5, "C/articles/evaluate-ai-agents-tools-outcomes.md": 4, "C/articles/maintain-llm-evaluation-suite.md": 3, "D/articles/choosing-model-benchmark.md": 4, "D/articles/public-benchmark-literacy.md": 4, "D/articles/build-task-eval.md": 3, "D/articles/datasets-holdouts-contamination.md": 3, "D/articles/metrics-rubrics.md": 4, "D/articles/human-evaluation-model-judges.md": 3, "D/articles/repeatability-uncertainty.md": 3, "D/articles/fair-model-comparisons.md": 3, "D/articles/latency-cost.md": 3, "D/articles/agent-system-evaluation.md": 3, "D/articles/maintaining-evaluations.md": 3 }, "bibliographic_associations": [ { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://arxiv.org/abs/2308.03688", "source_id": "agentbench", "source_title": "AgentBench: Evaluating LLMs as Agents", "checked_author_year": "Xiao Liu et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://arxiv.org/abs/2402.03927", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "checked_author_year": "Simone Balloccu et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://arxiv.org/abs/2406.07791", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "checked_author_year": "Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma, Soroush Vosoughi; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_id": "sweguide", "source_title": "Evaluation Guide", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/01-benchmarking-models.md", "url": "https://www.swebench.com/original.html", "source_id": "sweoriginal", "source_title": "SWE-bench", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/02-public-benchmarks.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/02-public-benchmarks.md", "url": "https://arxiv.org/abs/2402.03927", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "checked_author_year": "Simone Balloccu et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/02-public-benchmarks.md", "url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_id": "sweguide", "source_title": "Evaluation Guide", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/02-public-benchmarks.md", "url": "https://www.swebench.com/original.html", "source_id": "sweoriginal", "source_title": "SWE-bench", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/03-task-design.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/04-datasets-holdouts.md", "url": "https://arxiv.org/abs/2402.03927", "source_id": "balloccu", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "checked_author_year": "Simone Balloccu et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/05-metrics-rubrics.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/06-human-model-judges.md", "url": "https://arxiv.org/abs/2406.07791", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "checked_author_year": "Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma, Soroush Vosoughi; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/07-repeatability-uncertainty.md", "url": "https://itl.nist.gov/div898/software/dataplot/refman2/auxillar/exacbici.htm", "source_id": "nistexact", "source_title": "Exact Binomial Confidence Limits", "checked_author_year": "NIST; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/08-fair-comparisons.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/10-agent-evaluation.md", "url": "https://arxiv.org/abs/2308.03688", "source_id": "agentbench", "source_title": "AgentBench: Evaluating LLMs as Agents", "checked_author_year": "Xiao Liu et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/10-agent-evaluation.md", "url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_id": "sweguide", "source_title": "Evaluation Guide", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "A/articles/11-results-maintenance.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2311.04850", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "checked_author_year": "Shuo Yang et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2311.09783", "source_id": "deng", "source_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "checked_author_year": "Chunyuan Deng et al.; 2023 preprint; NAACL 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2403.07974", "source_id": "livecode", "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "checked_author_year": "Naman Jain et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2407.01502", "source_id": "kapoor", "source_title": "AI Agents That Matter", "checked_author_year": "Sayash Kapoor et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2502.14425", "source_id": "cheng", "source_title": "A Survey on Data Contamination for Large Language Models", "checked_author_year": "Yuxing Cheng, Yi Chang, Yuan Wu; 2025", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/contamination-leakage-and-holdouts.md", "url": "https://arxiv.org/abs/2504.20879", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "checked_author_year": "Shivalika Singh et al.; 2025", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "url": "https://arxiv.org/abs/2010.06595", "source_id": "card", "source_title": "With Little Power Comes Great Responsibility", "checked_author_year": "Dallas Card et al.; 2020", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "url": "https://arxiv.org/abs/2404.12272", "source_id": "shankar", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "checked_author_year": "Shreya Shankar et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "url": "https://arxiv.org/abs/2411.00640", "source_id": "miller", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "checked_author_year": "Evan Miller; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "url": "https://arxiv.org/abs/2411.12990", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "checked_author_year": "Anka Reuel et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://arxiv.org/abs/2310.06770", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "checked_author_year": "Carlos E. Jimenez et al.; 2023 preprint; ICLR 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://arxiv.org/abs/2311.12983", "source_id": "gaia", "source_title": "GAIA: a benchmark for General AI Assistants", "checked_author_year": "Gr\u00e9goire Mialon et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://arxiv.org/abs/2403.07974", "source_id": "livecode", "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "checked_author_year": "Naman Jain et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://arxiv.org/abs/2406.12045", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://arxiv.org/abs/2407.01502", "source_id": "kapoor", "source_title": "AI Agents That Matter", "checked_author_year": "Sayash Kapoor et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_id": "agentevals", "source_title": "Evaluate agent workflows", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://inspect.aisi.org.uk/", "source_id": "inspect", "source_title": "Inspect", "checked_author_year": "UK AI Security Institute / Meridian Labs; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-agents-and-tool-use.md", "url": "https://www.swebench.com/", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2009.03300", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "checked_author_year": "Dan Hendrycks et al.; 2020 preprint; ICLR 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2206.04615", "source_id": "bigbench", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "checked_author_year": "Aarohi Srivastava et al.; 2022", "verdict": "bibliographic_label_issue", "issues": [ "Preprint2022 is valid; TMLR2022 venue-year label is incorrect. Author-hosted journal PDF header says May2023 (https://joshrule.com/files/srivastava2023beyond.pdf)." ], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2310.06770", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "checked_author_year": "Carlos E. Jimenez et al.; 2023 preprint; ICLR 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2311.04850", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "checked_author_year": "Shuo Yang et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2311.12983", "source_id": "gaia", "source_title": "GAIA: a benchmark for General AI Assistants", "checked_author_year": "Gr\u00e9goire Mialon et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2403.04132", "source_id": "arena", "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "checked_author_year": "Wei-Lin Chiang et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2406.12045", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2407.01502", "source_id": "kapoor", "source_title": "AI Agents That Matter", "checked_author_year": "Sayash Kapoor et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2411.00640", "source_id": "miller", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "checked_author_year": "Evan Miller; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2411.12990", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "checked_author_year": "Anka Reuel et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://arxiv.org/abs/2504.20879", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "checked_author_year": "Shivalika Singh et al.; 2025", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "url": "https://www.swebench.com/", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/human-evaluation-and-model-judges.md", "url": "https://arxiv.org/abs/2306.05685", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "checked_author_year": "Lianmin Zheng et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/human-evaluation-and-model-judges.md", "url": "https://arxiv.org/abs/2404.12272", "source_id": "shankar", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "checked_author_year": "Shreya Shankar et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/human-evaluation-and-model-judges.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "url": "https://arxiv.org/abs/2411.12990", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "checked_author_year": "Anka Reuel et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "url": "https://arxiv.org/abs/2504.20879", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "checked_author_year": "Shivalika Singh et al.; 2025", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://ai.google.dev/gemini-api/docs/pricing", "source_id": "gemini", "source_title": "Gemini Developer API pricing", "checked_author_year": "Google; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://arxiv.org/abs/2407.01502", "source_id": "kapoor", "source_title": "AI Agents That Matter", "checked_author_year": "Sayash Kapoor et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://claude.com/pricing", "source_id": "claude", "source_title": "Plans and pricing", "checked_author_year": "Anthropic; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_id": "latency", "source_title": "Latency optimization", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://developers.openai.com/api/docs/guides/reasoning", "source_id": "reasoning", "source_title": "Reasoning models", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/latency-token-usage-and-cost.md", "url": "https://developers.openai.com/api/docs/pricing", "source_id": "pricing", "source_title": "Pricing", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/objective-metrics-and-rubrics.md", "url": "https://arxiv.org/abs/2107.03374", "source_id": "humaneval", "source_title": "Evaluating Large Language Models Trained on Code", "checked_author_year": "Mark Chen et al.; 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/objective-metrics-and-rubrics.md", "url": "https://arxiv.org/abs/2306.05685", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "checked_author_year": "Lianmin Zheng et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/objective-metrics-and-rubrics.md", "url": "https://arxiv.org/abs/2310.06770", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "checked_author_year": "Carlos E. Jimenez et al.; 2023 preprint; ICLR 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/objective-metrics-and-rubrics.md", "url": "https://arxiv.org/abs/2406.12045", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/objective-metrics-and-rubrics.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "url": "https://developers.openai.com/api/docs/guides/reasoning", "source_id": "reasoning", "source_title": "Reasoning models", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "url": "https://inspect.aisi.org.uk/", "source_id": "inspect", "source_title": "Inspect", "checked_author_year": "UK AI Security Institute / Meridian Labs; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "url": "https://www.swebench.com/", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2009.03300", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "checked_author_year": "Dan Hendrycks et al.; 2020 preprint; ICLR 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2206.04615", "source_id": "bigbench", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "checked_author_year": "Aarohi Srivastava et al.; 2022", "verdict": "bibliographic_label_issue", "issues": [ "Preprint2022 is valid; TMLR2022 venue-year label is incorrect. Author-hosted journal PDF header says May2023 (https://joshrule.com/files/srivastava2023beyond.pdf)." ], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2403.04132", "source_id": "arena", "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "checked_author_year": "Wei-Lin Chiang et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2411.12990", "source_id": "betterbench", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "checked_author_year": "Anka Reuel et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "url": "https://arxiv.org/abs/2504.20879", "source_id": "illusion", "source_title": "The Leaderboard Illusion", "checked_author_year": "Shivalika Singh et al.; 2025", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "url": "https://arxiv.org/abs/2010.06595", "source_id": "card", "source_title": "With Little Power Comes Great Responsibility", "checked_author_year": "Dallas Card et al.; 2020", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "url": "https://arxiv.org/abs/2107.03374", "source_id": "humaneval", "source_title": "Evaluating Large Language Models Trained on Code", "checked_author_year": "Mark Chen et al.; 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "url": "https://arxiv.org/abs/2406.10229", "source_id": "madaan", "source_title": "Quantifying Variance in Evaluation Benchmarks", "checked_author_year": "Lovish Madaan et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "url": "https://arxiv.org/abs/2406.12045", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "url": "https://arxiv.org/abs/2411.00640", "source_id": "miller", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "checked_author_year": "Evan Miller; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_id": "latency", "source_title": "Latency optimization", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "url": "https://developers.openai.com/api/docs/guides/prompt-caching", "source_id": "caching", "source_title": "Prompt caching", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "url": "https://developers.openai.com/api/docs/pricing", "source_id": "pricing", "source_title": "Pricing", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/ai-model-benchmarking-guide.md", "url": "https://arxiv.org/abs/2009.03300", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "checked_author_year": "Dan Hendrycks et al.; 2020 preprint; ICLR 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/ai-model-benchmarking-guide.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_id": "bootstrap", "source_title": "scipy.stats.bootstrap", "checked_author_year": "SciPy developers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm", "source_id": "nist", "source_title": "7.2.4.1. Confidence intervals", "checked_author_year": "NIST/SEMATECH; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/design-llm-evaluation-tasks.md", "url": "https://aclanthology.org/2020.acl-main.442/", "source_id": "checklist", "source_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList", "checked_author_year": "Marco Tulio Ribeiro, Tongshuang Wu, Carlos Guestrin, Sameer Singh; 2020", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "url": "https://arxiv.org/abs/2406.12045", "source_id": "tau", "source_title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "url": "https://arxiv.org/html/2406.12045v1#S3", "source_id": "taumetric", "source_title": "\u03c4-bench, section 3", "checked_author_year": "Shunyu Yao et al.; 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "url": "https://arxiv.org/abs/1803.09010", "source_id": "datasheets", "source_title": "Datasheets for Datasets", "checked_author_year": "Timnit Gebru et al.; 2018 preprint; CACM 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "url": "https://arxiv.org/abs/2310.16789", "source_id": "shi", "source_title": "Detecting Pretraining Data from Large Language Models", "checked_author_year": "Weijia Shi et al.; 2023 preprint; revised 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "url": "https://arxiv.org/abs/2310.17623", "source_id": "oren", "source_title": "Proving Test Set Contamination in Black Box Language Models", "checked_author_year": "Yonatan Oren et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_id": "skleak", "source_title": "Common pitfalls and recommended practices", "checked_author_year": "scikit-learn developers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/fair-ai-model-comparisons.md", "url": "https://arxiv.org/abs/2310.11324", "source_id": "sclar", "source_title": "Quantifying Language Models\u2019 Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting", "checked_author_year": "Melanie Sclar et al.; 2023 preprint; revised 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/fair-ai-model-comparisons.md", "url": "https://developers.openai.com/api/docs/guides/reasoning", "source_id": "reasoning", "source_title": "Reasoning models", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/fair-ai-model-comparisons.md", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "source_id": "harness", "source_title": "Language Model Evaluation Harness", "checked_author_year": "EleutherAI; living repository; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/human-reviewers-llm-judges.md", "url": "https://arxiv.org/abs/2306.05685", "source_id": "mtbench", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "checked_author_year": "Lianmin Zheng et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_id": "skmetrics", "source_title": "precision_recall_fscore_support", "checked_author_year": "scikit-learn developers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/maintain-llm-evaluation-suite.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "url": "https://arxiv.org/abs/2107.03374", "source_id": "humaneval", "source_title": "Evaluating Large Language Models Trained on Code", "checked_author_year": "Mark Chen et al.; 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "url": "https://arxiv.org/html/2107.03374v2#S2.SS1", "source_id": "passk", "source_title": "Evaluating Large Language Models Trained on Code, section 2.1", "checked_author_year": "Mark Chen et al.; 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "url": "https://www.swebench.com/", "source_id": "sweboard", "source_title": "SWE-bench Leaderboards", "checked_author_year": "SWE-bench maintainers; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/agent-system-evaluation.md", "url": "https://arxiv.org/abs/2609.08149", "source_id": "pro", "source_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents", "checked_author_year": "Pujun Zheng et al.; 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/agent-system-evaluation.md", "url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_id": "agentevals", "source_title": "Evaluate agent workflows", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/choosing-model-benchmark.md", "url": "https://arxiv.org/abs/2009.03300", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "checked_author_year": "Dan Hendrycks et al.; 2020 preprint; ICLR 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/choosing-model-benchmark.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/choosing-model-benchmark.md", "url": "https://arxiv.org/abs/2310.06770", "source_id": "swepaper", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "checked_author_year": "Carlos E. Jimenez et al.; 2023 preprint; ICLR 2024", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/choosing-model-benchmark.md", "url": "https://developers.openai.com/api/docs/pricing", "source_id": "pricing", "source_title": "Pricing", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/datasets-holdouts-contamination.md", "url": "https://arxiv.org/abs/2311.04850", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "checked_author_year": "Shuo Yang et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/datasets-holdouts-contamination.md", "url": "https://arxiv.org/abs/2609.02899", "source_id": "xiao", "source_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards", "checked_author_year": "Xingyao Xiao and Yihong Cheng; 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/human-evaluation-model-judges.md", "url": "https://arxiv.org/abs/2406.07791", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "checked_author_year": "Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma, Soroush Vosoughi; 2024", "verdict": "bibliographic_label_issue", "issues": [ "Inline author label Zheng is incorrect; Lin Shi et al. is correct. Year2024 is initial-preprint year, not a fabricated date." ], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/human-evaluation-model-judges.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/latency-cost.md", "url": "https://developers.openai.com/api/docs/pricing", "source_id": "pricing", "source_title": "Pricing", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/maintaining-evaluations.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/metrics-rubrics.md", "url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_id": "eval", "source_title": "Evaluation best practices", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/metrics-rubrics.md", "url": "https://developers.openai.com/api/docs/guides/graders", "source_id": "graders", "source_title": "Graders", "checked_author_year": "OpenAI; living page; accessed 2026", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/public-benchmark-literacy.md", "url": "https://arxiv.org/abs/2009.03300", "source_id": "mmlu", "source_title": "Measuring Massive Multitask Language Understanding", "checked_author_year": "Dan Hendrycks et al.; 2020 preprint; ICLR 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/public-benchmark-literacy.md", "url": "https://arxiv.org/abs/2211.09110", "source_id": "helm", "source_title": "Holistic Evaluation of Language Models", "checked_author_year": "Percy Liang et al.; 2022", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/public-benchmark-literacy.md", "url": "https://arxiv.org/abs/2311.04850", "source_id": "yang", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "checked_author_year": "Shuo Yang et al.; 2023", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/public-benchmark-literacy.md", "url": "https://arxiv.org/abs/2406.07791", "source_id": "position", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "checked_author_year": "Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma, Soroush Vosoughi; 2024", "verdict": "bibliographic_label_issue", "issues": [ "Inline author label Zheng is incorrect; Lin Shi et al. is correct. Year2024 is initial-preprint year, not a fabricated date." ], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." }, { "article_file": "D/articles/repeatability-uncertainty.md", "url": "https://arxiv.org/abs/2103.03098", "source_id": "bouthillier", "source_title": "Accounting for Variance in Machine Learning Benchmarks", "checked_author_year": "Xavier Bouthillier et al.; 2021", "verdict": "association_matches", "issues": [], "retrieval_date": "2026-10-28", "scope": "Title/URL and claimed author/year association; full author list not reproduced. Living documentation access year is not represented as its original publication year." } ], "bibliographic_association_count": 127, "limits": [ "Sampling is not exhaustive fact verification. Unchecked claims include every soft editorial causal assertion, all suggested team-specific thresholds/cadences, all legal/account-specific suitability and complete paper methodologies.", "No hidden training data, raw benchmark reruns, empirical workload measurements, live API experiments, private terms or invoices inspected.", "Live verification on28October cannot independently certify that a page was unchanged on22or25October.", "Reviewer source retrieval: OpenReview challenge blocked journal PDF, resolved via author-hosted original journal PDF; two guessed SQuAD repository paths404 and maintainer script inaccessible. Token-F1 issue assessed from precision/recall definitions and counterexample, not the unavailable script.", "One judge, one submission run per candidate. No model identities, usage, runtime, historical scores or coordinator materials accessed." ], "fabricated_evidence_findings": [], "all_discovered_material_dynamic_claims_checked": true }