{ "status": "complete_stage1_claim_checks", "access_precision": "UTC calendar date; retrieval tool responses are preserved separately. Exact per-request wall-clock times were not captured.", "claims": [ { "id": "A-001", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L22", "claim": "MMLU covers 57 academic/professional tasks", "source_url": "https://arxiv.org/abs/2009.03300", "source_title": "Measuring Massive Multitask Language Understanding", "source_date": "2020-09-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract, task breadth and intended knowledge/problem-solving construct.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-002", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L22", "claim": "BIG-bench: 204 tasks, 450 authors, 132 institutions and expert raters", "source_url": "https://arxiv.org/abs/2206.04615", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "source_date": "2022-06-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract, benchmark construction and human baseline.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-003", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L22", "claim": "HELM: seven metrics, 16 core and 42 total scenarios", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_qualification", "supporting_evidence": "Abstract supports counts; seven metrics were applied to core scenarios where feasible, not uniformly to all 42.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-004", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L26", "claim": "Arena had more than 240,000 votes and crowd/expert agreement in the 2024 paper", "source_url": "https://arxiv.org/abs/2403.04132", "source_title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "source_date": "2024-03-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract; explicitly historical.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-005", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L26", "claim": "BetterBench examined 24 benchmarks against 46 practices and found significance/replication gaps", "source_url": "https://arxiv.org/abs/2411.12990", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_date": "2024-11-20", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes sample, framework and findings.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-006", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "substantive_factual", "location": "L36", "claim": "SWE-bench original size is 2,294 issues from 12 Python repositories", "source_url": "https://arxiv.org/abs/2310.06770", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_date": "2023-10-10", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract identifies dataset and task.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-007", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "changing_benchmark_specification", "location": "L36", "claim": "Verified has 500 instances; Bash Only uses mini-SWE-agent", "source_url": "https://www.swebench.com/", "source_title": "SWE-bench Leaderboards", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer site sections Verified and Bash Only.", "limits": "Current maintainer view corroborates the dated observation; no archived 25 October snapshot was retrieved.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-008", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "empirical_result", "location": "L36", "claim": "GAIA reported 92% human versus 15% GPT-4-with-plugins", "source_url": "https://arxiv.org/abs/2311.12983", "source_title": "GAIA: a benchmark for General AI Assistants", "source_date": "2023-11-21", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract, original study conditions; not a current model ranking.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-009", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "empirical_result", "location": "L36", "claim": "tau-bench: database goal-state comparison; under 50% single-run and pass^8 under 25% retail", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports method and original agent results.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-010", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "empirical_result", "location": "L64", "claim": "Rephrasing bypasses n-gram decontamination and 8–18% HumanEval overlap was found", "source_url": "https://arxiv.org/abs/2311.04850", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_date": "2023-11-08", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports rephrasing mechanism and named-corpus overlap.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-011", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "hypothetical_arithmetic", "location": "L90", "claim": "Hypothetical paired SE about 3.9 points, CI about −3.6 to +11.6 and about 900 tasks", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Independent arithmetic: variances .78*.22 and .74*.26, correlation .5 produce SE .03897; power formula gives about 894 tasks.", "limits": "Illustrative assumed correlation, approximate z-based planning, not a measured result or exact count-compatible dataset.", "producer_research_date": "2026-10-25" }, { "id": "A-012", "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "claim_type": "substantive_factual", "location": "L22", "claim": "MMLU has 57 tasks and original models lagged expert accuracy on each", "source_url": "https://arxiv.org/abs/2009.03300", "source_title": "Measuring Massive Multitask Language Understanding", "source_date": "2020-09-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract supports task scope and historical gap.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-013", "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "claim_type": "substantive_factual", "location": "L22", "claim": "BIG-bench performance/calibration improved with scale but remained poor; brittle metrics affected breakthroughs", "source_url": "https://arxiv.org/abs/2206.04615", "source_title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "source_date": "2022-06-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract findings, not inference about current models.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-014", "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "claim_type": "substantive_factual", "location": "L24", "claim": "HELM: 30 models, 16 core/26 targeted, 17.9% to 96.0% coverage", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes scenario coverage and before/after comparability.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-015", "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "claim_type": "empirical_result", "location": "L40", "claim": "Arena data shares: 19.2%, 20.4%, and 29.7% for combined 83 open-weight models", "source_url": "https://arxiv.org/abs/2504.20879", "source_title": "The Leaderboard Illusion", "source_date": "2025-04-29", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports these estimates with the same attribution and caveats.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-016", "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "claim_type": "substantive_factual", "location": "L42", "claim": "Most benchmarks in BetterBench did not report significance or allow easy replication", "source_url": "https://arxiv.org/abs/2411.12990", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_date": "2024-11-20", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports this for the examined sample, not all benchmarks universally.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-017", "cluster": "A", "article_file": "articles/designing-evaluation-tasks-and-datasets.md", "claim_type": "methodological", "location": "L26", "claim": "Evaluation set should include typical, edge and adversarial cases", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official evaluation guide, dataset design examples and testing principles.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-018", "cluster": "A", "article_file": "articles/designing-evaluation-tasks-and-datasets.md", "claim_type": "methodological", "location": "L102", "claim": "Criteria drift can emerge while reviewers see outputs", "source_url": "https://arxiv.org/abs/2404.12272", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_date": "2024-04-18", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract explicitly defines this dependency.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-019", "cluster": "A", "article_file": "articles/designing-evaluation-tasks-and-datasets.md", "claim_type": "methodological", "location": "L106–114", "claim": "Sample size depends on variance/effect/correlation and correlated tasks reduce independent information", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Sections 2.2, 4.2 and 5 give clustered/paired and power formulas.", "limits": "The article’s 50–100/300–500/1,000 ranges are editorial heuristics, not universal guarantees or direct empirical results.", "producer_research_date": "2026-10-25" }, { "id": "A-020", "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "claim_type": "empirical_result", "location": "L36", "claim": "Rephrased/translated test data can evade n-gram filtering; 13B overfit and 8–18% overlap", "source_url": "https://arxiv.org/abs/2311.04850", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_date": "2023-11-08", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports all of these, on specified benchmarks/corpora.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-021", "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "claim_type": "empirical_result", "location": "L38", "claim": "TS-Guessing masks a wrong answer; MMLU rates 52% and 57%", "source_url": "https://arxiv.org/abs/2311.09783", "source_title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "source_date": "2023-11-16", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract defines procedure and reports exact-match rates; draft appropriately stops short of proof.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-022", "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "claim_type": "substantive_factual", "location": "L40", "claim": "Survey separates white/grey/black-box detection and updating/rewriting/prevention", "source_url": "https://arxiv.org/abs/2502.14425", "source_title": "A Survey on Data Contamination for Large Language Models", "source_date": "2025-02-20", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original survey abstract gives the taxonomy.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-023", "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "claim_type": "changing_benchmark_specification", "location": "L44", "claim": "LiveCodeBench continuously collects competition problems and reports 400 May 2023–May 2024 problems", "source_url": "https://arxiv.org/abs/2403.07974v1", "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "source_date": "2024-03-12", "accessed_at_utc": "2026-10-29", "verdict": "partly_contradicted", "supporting_evidence": "Original March 2024 v1 abstract gives May 2023–February 2024; revised June 2024 abstract gives May 2023–May 2024. The draft conflates the revised range with the initial release.", "limits": "Ordinary historical/version correction; the source exists and its revised count is real. No fabricated evidence alleged.", "producer_research_date": "2026-10-25" }, { "id": "A-024", "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "claim_type": "methodological", "location": "L88", "claim": "Unclear provider decontamination means the score is an upper bound on likely performance", "source_url": "https://arxiv.org/abs/2311.04850", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_date": "2023-11-08", "accessed_at_utc": "2026-10-29", "verdict": "unsupported_guidance", "supporting_evidence": "The source shows contamination can inflate scores; it does not establish a mathematical upper bound for an arbitrary deployment distribution.", "limits": "Unverified provider disclosure warrants uncertainty, not a numeric bound. No fabricated source or measured result is alleged.", "producer_research_date": "2026-10-25" }, { "id": "A-025", "cluster": "A", "article_file": "articles/objective-metrics-and-rubrics.md", "claim_type": "methodological", "location": "L30", "claim": "pass@k estimator is 1−C(n−c,k)/C(n,k)", "source_url": "https://arxiv.org/abs/2107.03374", "source_title": "Evaluating Large Language Models Trained on Code", "source_date": "2021-07-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original full paper section 2.1 equation 1; requires n≥k and independent samples.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-026", "cluster": "A", "article_file": "articles/objective-metrics-and-rubrics.md", "claim_type": "methodological", "location": "L32", "claim": "pass^k evaluates all k attempts succeeding", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original benchmark’s reliability metric; distinct from any-success pass@k.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-027", "cluster": "A", "article_file": "articles/objective-metrics-and-rubrics.md", "claim_type": "methodological", "location": "L38", "claim": "Tool choice and argument data precision are distinct evaluation targets", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide architecture tables distinguish these dimensions.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-028", "cluster": "A", "article_file": "articles/human-evaluation-and-model-judges.md", "claim_type": "methodological", "location": "L38", "claim": "Reviewers can discover criteria while grading outputs", "source_url": "https://arxiv.org/abs/2404.12272", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_date": "2024-04-18", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract’s criteria-drift finding.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-029", "cluster": "A", "article_file": "articles/human-evaluation-and-model-judges.md", "claim_type": "empirical_result", "location": "L72", "claim": "GPT-4 judge agreement exceeded 80% under the MT-bench/Arena study conditions", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes controlled/crowdsourced agreement and human agreement baseline.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-030", "cluster": "A", "article_file": "articles/human-evaluation-and-model-judges.md", "claim_type": "methodological", "location": "L72", "claim": "LLM judges have position, verbosity and self-enhancement biases and limited reasoning", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract enumerates these limitations.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-031", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "methodological", "location": "L28", "claim": "Training-seed variance and simple format changes can move benchmark evaluation", "source_url": "https://arxiv.org/abs/2406.10229", "source_title": "Quantifying Variance in Evaluation Benchmarks", "source_date": "2024-06-14", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract measures seed variance/monotonicity and choice-to-completion changes.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-032", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "methodological", "location": "L32–42", "claim": "Mean SE=sd/sqrt(n), Bernoulli SE and approximate 1.96-SE intervals", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Full paper section 2.1 equations 1–3 with independent-question and asymptotic assumptions.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-033", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "hypothetical_arithmetic", "location": "L50–52", "claim": "Hypothetical correlation .5 gives SE about 3.9 points and roughly 900 tasks", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Calculated from the displayed Bernoulli variances and paired/power formula: SE=.03897, CI≈[−.0364,.1164], n≈894.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-034", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "empirical_result", "location": "L58", "claim": "Cluster-adjusted SE can be about three times naive SE", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Table 4 reports a 3.05 ratio for DROP; not a universal design effect.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-035", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "methodological", "location": "L66", "claim": "Two/four repeats reduce total variance about one-third/half in a simple example", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Section 3.1 uniform-difficulty binary example; draft explicitly says simple example.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-036", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "empirical_result", "location": "L82", "claim": "Typical 2,000-sentence MT sets had around 75% power to detect 1 BLEU", "source_url": "https://arxiv.org/abs/2010.06595", "source_title": "With Little Power Comes Great Responsibility", "source_date": "2020-10-13", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports that study-specific result.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-037", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "hypothetical_arithmetic", "location": "L88", "claim": "Miller’s assumptions imply approximately 969 questions for a .03 effect", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Section 5 equation 9 and worked example; arithmetic 7.85*(1/9)/.03²≈969.", "limits": "Verifies the paper’s own illustrative example, not general sample-size validity; the paper’s preceding covariance example has an arithmetic inconsistency, so use estimated variance rather than its asserted generic variance.", "producer_research_date": "2026-10-25" }, { "id": "A-038", "cluster": "A", "article_file": "articles/prompt-tool-and-configuration-fairness.md", "claim_type": "methodological", "location": "L74", "claim": "Inspect makes evaluation datasets, solvers/scorers and sandboxes explicit", "source_url": "https://inspect.aisi.org.uk/", "source_title": "Inspect: Framework for Large Language Model Evaluations", "source_date": "2024-05", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official introduction describes composable components and sandbox support.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-039", "cluster": "A", "article_file": "articles/prompt-tool-and-configuration-fairness.md", "claim_type": "changing_benchmark_specification", "location": "L100", "claim": "SWE-bench Bash Only holds mini-SWE-agent environment constant", "source_url": "https://www.swebench.com/", "source_title": "SWE-bench Leaderboards", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer’s Bash Only description.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-040", "cluster": "A", "article_file": "articles/prompt-tool-and-configuration-fairness.md", "claim_type": "methodological", "location": "L110–117", "claim": "Higher effort always increases tokens/latency; equal nominal effort is matched effort", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "overstated_or_unsupported", "supporting_evidence": "Official guide describes adaptive/model-dependent effort. No per-request monotonic guarantee or cross-provider equivalence follows.", "limits": "Match observed resources/budgets and provider-supported controls. Incorrect fairness guidance requires substantive correction before publication.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-041", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "methodological", "location": "L20", "claim": "Accuracy-only agent evaluation can yield needless complexity/cost", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract identifies this failure and joint cost/accuracy optimisation.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-042", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "methodological", "location": "L36", "claim": "Output reduction can roughly reduce generation latency proportionally; input reduction can yield only 1–5%", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_title": "Latency optimization", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide, fewer output/input tokens sections; scoped heuristics, not promises.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-043", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "changing_model_specification", "location": "L45–47", "claim": "Reasoning tokens billed as output/context; 25,000-token initial buffer guidance", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official reasoning guide accounting and incomplete-response advice.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-044", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "changing_pricing", "location": "L72–73", "claim": "Astra $10/$50/$1 cached, batch $5/$25; Luna $.10/$.50/$.01 cached, batch $.05/$.25", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official standard and Flex/Batch rows agree.", "limits": "Live 29 October corroboration, not proof of an archived 25 October retrieval; USD per million tokens and tier-specific.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-045", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "changing_pricing", "location": "L74–75", "claim": "Fable 5.1 $10/$50, $.25 cache; Haiku 5.5 $.10/$.50 with cache from $.01", "source_url": "https://claude.com/platform/api", "source_title": "Claude Platform — API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_qualification", "supporting_evidence": "Official pricing agrees for Haiku prompts ≤100K; >100K has $.50/$2.50 and $.05 read.", "limits": "Article table must state the Haiku input-length tier. Current confirmation does not independently prove the earlier access date; batch-discount check remains pending. Reviewer verified the API-platform rate card; candidate pricing citation and reviewer verification URL are distinguished.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25", "candidate_citation_url": "https://claude.com/pricing" }, { "id": "A-046", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "changing_pricing", "location": "L76–78", "claim": "Gemini 4 Argon $2/$10/$.10 cache until 30 Nov; doubles 1 Dec; batch $1/$5", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "source_title": "Gemini Developer API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official pricing standard/batch sections and dated promotion match.", "limits": "Current official schedule corroborates dated observation; no historical archive retrieved.", "source_date_as_initially_catalogued": "2026-10-22", "producer_research_date": "2026-10-25" }, { "id": "A-047", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "hypothetical_arithmetic", "location": "L84–104", "claim": "Illustrative retry-adjusted costs about $.32/$.0032; cached-frontier cost about $.24", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Independent arithmetic: .25*1.1/.85=.323529; .0025*1.1/.85=.003235; (.0825+.1)*1.1/.85=.236176; incremental success cost (.275−.00275)*100/15=1.815.", "limits": "Arithmetic only. Assumed costs, success and attempts are hypothetical; not provider performance measurements. Use total billed output including reasoning in real accounting.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-048", "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "claim_type": "empirical_result", "location": "L37", "claim": "Original SWE-bench: 2,294 issues, 12 Python repos, early best 1.96%", "source_url": "https://arxiv.org/abs/2310.06770", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_date": "2023-10-10", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes dataset and Claude 2 result.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-049", "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "claim_type": "changing_benchmark_specification", "location": "L37", "claim": "Verified 500, Lite 300, Multilingual 300/42 repositories/nine languages, Bash Only common scaffold", "source_url": "https://www.swebench.com/", "source_title": "SWE-bench Leaderboards", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Current official subset descriptions match.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-050", "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "claim_type": "empirical_result", "location": "L39", "claim": "tau-bench database-goal oracle and original <50%/pass^8<25% result", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports method and historical results.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-051", "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "claim_type": "empirical_result", "location": "L41", "claim": "GAIA 466 questions, 300 held answers, historical 92% human/15% GPT-4 plugins", "source_url": "https://arxiv.org/abs/2311.12983", "source_title": "GAIA: a benchmark for General AI Assistants", "source_date": "2023-11-21", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract explicitly reports task/holdout counts and baseline outcomes.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-052", "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "claim_type": "methodological", "location": "L73", "claim": "Agent evaluation guide recommends traces for debugging then datasets for repeatability", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_title": "Evaluate agent workflows", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide headings and trace/dataset workflow.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25" }, { "id": "A-053", "cluster": "A", "article_file": "articles/interpreting-results-and-maintaining-evals.md", "claim_type": "substantive_factual", "location": "L45", "claim": "BetterBench found statistical-significance and replication reporting gaps", "source_url": "https://arxiv.org/abs/2411.12990", "source_title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices", "source_date": "2024-11-20", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract, bounded to evaluated benchmarks.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-054", "cluster": "A", "article_file": "articles/interpreting-results-and-maintaining-evals.md", "claim_type": "methodological", "location": "L75", "claim": "Private testing, selective disclosure and unequal Arena sampling can distort rankings", "source_url": "https://arxiv.org/abs/2504.20879", "source_title": "The Leaderboard Illusion", "source_date": "2025-04-29", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports these mechanisms; internal-team transfer is editorial inference.", "limits": "Source statement checked, not experimentally reproduced. Source version/research date remains distinct from retrieval date.", "producer_research_date": "2026-10-25" }, { "id": "A-055", "cluster": "A", "article_file": "articles/interpreting-results-and-maintaining-evals.md", "claim_type": "changing_platform_specification", "location": "L112", "claim": "OpenAI Evals becomes read-only 31 Oct 2026 and shuts down 30 Nov 2026", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations — Evals platform timeline", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live official deprecation notice at top of evaluation guide agrees exactly.", "limits": "Current scheduled dates; not an observed future shutdown. Earlier dated claim preserved. Reviewer cross-checked the dated deprecation table; the candidate guide citation is retained separately.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-25", "candidate_citation_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices" }, { "id": "A-056", "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "claim_type": "methodological", "location": "L34", "claim": "Each architecture introduces a new source of nondeterminism, including multi-step workflows", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "overstated", "supporting_evidence": "Guide workflow section explicitly says workflows introduce no new elements of nondeterminism, though they involve multiple model interactions. Agent tools/handoffs do add evaluation targets.", "limits": "Correction to attribution and precision; overall system-evaluation argument remains sound.", "producer_research_date": "2026-10-25" }, { "id": "A-057", "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "claim_type": "changing_pricing", "location": "L74–78", "claim": "Anthropic batch processing gives 50% discount", "source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "source_title": "Pricing — Claude Platform Docs", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official detailed pricing, Batch processing section, specifies 50 percent off both input/output and lists Fable/Haiku batch rows.", "limits": "Current 29 October confirmation; first-party asynchronous batch terms, not a blanket third-party/cloud offer.", "producer_research_date": "2026-10-25" }, { "id": "B-001", "cluster": "B", "article_file": "articles/01-benchmarking-models.md", "claim_type": "methodological", "location": "L23", "claim": "AgentBench evaluates interactive environments beyond static model answers", "source_url": "https://arxiv.org/abs/2308.03688", "source_title": "AgentBench: Evaluating LLMs as Agents", "source_date": "2023-08-07; revised 2025-10-04", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes eight interactive environments and multi-step decision-making.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-002", "cluster": "B", "article_file": "articles/01-benchmarking-models.md", "claim_type": "methodological", "location": "L29", "claim": "HELM compares multiple dimensions/scenarios including efficiency/robustness", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16; revised 2023-10-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes seven metrics and standardised scenario evaluation.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-003", "cluster": "B", "article_file": "articles/01-benchmarking-models.md", "claim_type": "methodological", "location": "L33", "claim": "SWE-bench uses issue-derived tasks and repository tests", "source_url": "https://www.swebench.com/original.html", "source_title": "SWE-bench — Original benchmark", "source_date": "undated live maintainer page", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer overview describes PR/issue collection and fail-to-pass tests.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-004", "cluster": "B", "article_file": "articles/01-benchmarking-models.md", "claim_type": "changing_benchmark_specification", "location": "L33", "claim": "Evaluation counts distinguish resolved, unresolved, incomplete and errors", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_title": "SWE-bench Evaluation Guide", "source_date": "undated live maintainer documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "What the counts mean table separates missing predictions, failed tests and ungraded errors.", "limits": "Live maintainer documentation agrees; no archived 28 October snapshot checked.", "producer_research_date": "2026-10-28" }, { "id": "B-005", "cluster": "B", "article_file": "articles/02-public-benchmarks.md", "claim_type": "methodological", "location": "L21", "claim": "SWE-bench resolves issues via specified repository tests", "source_url": "https://www.swebench.com/original.html", "source_title": "SWE-bench — Original benchmark", "source_date": "undated live maintainer page", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer evaluation overview gives issue inputs and fail-to-pass tests.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-006", "cluster": "B", "article_file": "articles/02-public-benchmarks.md", "claim_type": "methodological", "location": "L31", "claim": "HELM standardises scenarios and reports multiple metrics", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16; revised 2023-10-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract states common evaluation conditions and exposed trade-offs.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-007", "cluster": "B", "article_file": "articles/02-public-benchmarks.md", "claim_type": "methodological", "location": "L37", "claim": "Benchmark exposure can be indirect and does not itself prove memorisation", "source_url": "https://arxiv.org/abs/2402.03927", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_date": "2024-02-06; revised 2024-02-22", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract distinguishes indirect user-data exposure and limited closed-data observability.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-008", "cluster": "B", "article_file": "articles/03-task-design.md", "claim_type": "methodological", "location": "L23", "claim": "Task-specific sets should cover typical/edge/adversarial cases", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official dataset guidance explicitly recommends this mix and expert labellers.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-009", "cluster": "B", "article_file": "articles/03-task-design.md", "claim_type": "methodological", "location": "L43", "claim": "Oversampled risk slices need separate/weighted interpretation", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Sampling/estimand assumptions in sections 2 and 5: sample inference depends on target population; weighting statement follows arithmetic of a changed sampling distribution.", "limits": "Reviewer method check of sampling logic; the draft gives editorial guidance and has no near citation for this particular weighting point.", "producer_research_date": "2026-10-28" }, { "id": "B-010", "cluster": "B", "article_file": "articles/03-task-design.md", "claim_type": "methodological", "location": "L57", "claim": "Related examples need grouped partitioning to limit leakage/dependence", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Paper identifies inadequate holdouts and benchmark shortcuts; Miller section 2.2 separately verifies within-group dependence.", "limits": "Group-splitting is an editorial design response; benchmark findings do not prescribe this exact partition policy.", "producer_research_date": "2026-10-28" }, { "id": "B-011", "cluster": "B", "article_file": "articles/04-datasets-holdouts.md", "claim_type": "methodological", "location": "L21", "claim": "Closed-model indirect benchmark exposure/reproducibility risks", "source_url": "https://arxiv.org/abs/2402.03927", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_date": "2024-02-06; revised 2024-02-22", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract describes user-data leakage routes and observed reproducibility issues.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-012", "cluster": "B", "article_file": "articles/04-datasets-holdouts.md", "claim_type": "methodological", "location": "L25–27", "claim": "Group-related cases are dependent; grouping may leave few independent units", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Section 2.2 gives clustered-unit analysis and its dependence assumptions.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-013", "cluster": "B", "article_file": "articles/04-datasets-holdouts.md", "claim_type": "methodological", "location": "L37", "claim": "Exact matching misses paraphrases and cannot establish no contamination", "source_url": "https://arxiv.org/abs/2402.03927", "source_title": "Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs", "source_date": "2024-02-06; revised 2024-02-22", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Closed-corpus observability limits support caution; corroborated by Yang rephrasing paper retrieved in source-retrieval-4.json.", "limits": "No detection tool was executed on a dataset; recommendation is bounded and does not claim absence was proven.", "producer_research_date": "2026-10-28" }, { "id": "B-014", "cluster": "B", "article_file": "articles/05-metrics-rubrics.md", "claim_type": "methodological", "location": "L23", "claim": "Precision is fraction of flagged cases truly positive; recall is fraction of actual positives flagged", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html", "source_title": "scikit-learn precision_score", "source_date": "live v1.9.1 documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official precision and linked recall definitions give TP/(TP+FP) and TP/(TP+FN).", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-015", "cluster": "B", "article_file": "articles/05-metrics-rubrics.md", "claim_type": "methodological", "location": "L27", "claim": "Passing executable tests does not prove every semantic requirement", "source_url": "https://www.swebench.com/original.html", "source_title": "SWE-bench — Original benchmark", "source_date": "undated live maintainer page", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer scoring checks specified fail-to-pass tests; the stated limitation correctly avoids extrapolation.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-016", "cluster": "B", "article_file": "articles/05-metrics-rubrics.md", "claim_type": "methodological", "location": "L29", "claim": "Evaluation methods can combine metric-based/human/model scoring; generic metrics alone are inadequate", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official evaluation guide evaluator types and anti-pattern guidance.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-017", "cluster": "B", "article_file": "articles/06-human-model-judges.md", "claim_type": "empirical_result", "location": "L31", "claim": "Pairwise judges show position bias varying across tested judges/tasks", "source_url": "https://arxiv.org/abs/2406.07791", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_date": "2024-06-12; revised 2025-11-11", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original study abstract reports position bias and tested-factor dependence.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-018", "cluster": "B", "article_file": "articles/06-human-model-judges.md", "claim_type": "methodological", "location": "L33", "claim": "Judge may prefer verbosity or own-family outputs", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09; revised 2023-12-24", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original MT-bench paper explicitly documents verbosity/self-enhancement biases.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-019", "cluster": "B", "article_file": "articles/06-human-model-judges.md", "claim_type": "methodological", "location": "L37–39", "claim": "Judge calibration requires independent human labels and error inspection beyond aggregate agreement", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official maintain-agreement/calibration advice; draft adds a sensible held-out validation protocol.", "limits": "Confusion-matrix and split protocol are editorial extensions, not attributed results from the cited position-bias paper.", "producer_research_date": "2026-10-28" }, { "id": "B-020", "cluster": "B", "article_file": "articles/07-repeatability-uncertainty.md", "claim_type": "methodological", "location": "L29", "claim": "Paired bootstrap resamples units with replacement while keeping candidate pairs together", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_title": "SciPy bootstrap", "source_date": "live v1.18.0 documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official procedure and paired parameter explain shared resampled indices.", "limits": "Group-level extension requires independent groups; draft notes few-group instability and selection bias. No bootstrap was run on actual candidate outputs.", "producer_research_date": "2026-10-28" }, { "id": "B-021", "cluster": "B", "article_file": "articles/07-repeatability-uncertainty.md", "claim_type": "methodological", "location": "L31", "claim": "Normal binomial approximations can be inaccurate for small n/few failures", "source_url": "https://itl.nist.gov/div898/software/dataplot/refman2/auxillar/exacbici.htm", "source_title": "Exact Binomial Confidence Limits", "source_date": "undated official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "NIST description states these exact limitations and an exact alternative.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-022", "cluster": "B", "article_file": "articles/07-repeatability-uncertainty.md", "claim_type": "methodological", "location": "L37", "claim": "Many related questions can overstate independent evidence", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Section 2.2 explains correlated groups and cluster-level information.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-023", "cluster": "B", "article_file": "articles/08-fair-comparisons.md", "claim_type": "methodological", "location": "L23", "claim": "Common scenarios and documented conditions improve comparability", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16; revised 2023-10-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original paper describes standardised multi-model evaluation.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-024", "cluster": "B", "article_file": "articles/08-fair-comparisons.md", "claim_type": "methodological", "location": "L31", "claim": "Identity-blind common graders still need human calibration and bias checks", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide calibration principles; MT-bench retrieved evidence independently supports style biases.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-025", "cluster": "B", "article_file": "articles/08-fair-comparisons.md", "claim_type": "methodological", "location": "L35", "claim": "Attempt/tool budgets confound model attribution and total resources matter", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract identifies conflated model/downstream evaluations and cost-sensitive comparison.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-026", "cluster": "B", "article_file": "articles/09-latency-cost.md", "claim_type": "methodological", "location": "L23", "claim": "High latency percentiles expose tail performance hidden by a typical value", "source_url": "https://sre.google/sre-book/monitoring-distributed-systems/", "source_title": "Monitoring Distributed Systems — Google SRE", "source_date": "2016 book; live text", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Google engineers describe latency tails and saturation-sensitive percentile monitoring.", "limits": "Reviewer-supplied primary support; the article has no external citation for this material measurement guidance.", "producer_research_date": "2026-10-28" }, { "id": "B-027", "cluster": "B", "article_file": "articles/09-latency-cost.md", "claim_type": "methodological", "location": "L27", "claim": "Displayed answer may omit billed reasoning work; cached categories/tool fees matter", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official reasoning guide bills invisible reasoning as output; pricing sources separately show cached/tool rates.", "limits": "No current provider rate is asserted in this article. Add a nearby accounting source for readers.", "producer_research_date": "2026-10-28" }, { "id": "B-028", "cluster": "B", "article_file": "articles/09-latency-cost.md", "claim_type": "hypothetical_arithmetic", "location": "L29–31", "claim": "Hypothetical $.0072 per attempt, $72 for 10,000 and $.009 per 8,000 accepted successes", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live pricing", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Recalculation: (2000*2+400*8)/1e6=.0072; *10000=72; /8000=.009.", "limits": "No external price claim: rates are expressly invented. Pricing URL supports unit conventions only; arithmetic was independently calculated.", "producer_research_date": "2026-10-28" }, { "id": "B-029", "cluster": "B", "article_file": "articles/10-agent-evaluation.md", "claim_type": "methodological", "location": "L15", "claim": "AgentBench evaluates agents in multiple interactive environments", "source_url": "https://arxiv.org/abs/2308.03688", "source_title": "AgentBench: Evaluating LLMs as Agents", "source_date": "2023-08-07; revised 2025-10-04", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original benchmark abstract describes this construct.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-030", "cluster": "B", "article_file": "articles/10-agent-evaluation.md", "claim_type": "methodological", "location": "L29", "claim": "Final completion prose does not prove an action; inspect trace and final state", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_title": "Evaluate agent workflows", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official trace guide describes model/tool-call records and scoring workflow violations.", "limits": "Final-state authority gate and exact expense scenarios are editorial recommendations; not measured benchmarks.", "producer_research_date": "2026-10-28" }, { "id": "B-031", "cluster": "B", "article_file": "articles/10-agent-evaluation.md", "claim_type": "changing_benchmark_specification", "location": "L31", "claim": "SWE-bench distinguishes completed/unresolved/error conditions", "source_url": "https://www.swebench.com/SWE-bench/guides/evaluation/", "source_title": "SWE-bench Evaluation Guide", "source_date": "undated live maintainer documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official result-field definitions match.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-032", "cluster": "B", "article_file": "articles/11-results-maintenance.md", "claim_type": "methodological", "location": "L41", "claim": "Continuous evaluation and human-calibrated scoring are recommended", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live official documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official tips and design-workflow sections recommend both.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-033", "cluster": "B", "article_file": "articles/11-results-maintenance.md", "claim_type": "methodological", "location": "L39", "claim": "Incident-derived cases retain shared origin rather than becoming independent evidence", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Independent-question/cluster framework verifies the stated dependence concern.", "limits": "Source text checked; no study or candidate experiment reproduced. Editorial transfer/recommendations are not empirical results.", "producer_research_date": "2026-10-28" }, { "id": "B-034", "cluster": "B", "article_file": "articles/11-results-maintenance.md", "claim_type": "methodological", "location": "L45–49", "claim": "Changing rubric/dataset/configuration changes what a score means", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16; revised 2023-10-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Standardised-scenario methodological design supports comparability requirements; official eval guide continuous evaluation supports change testing.", "limits": "Preserving/rescoring versions is editorial operational guidance; source does not validate this exact maintenance protocol.", "producer_research_date": "2026-10-28" }, { "id": "C-001", "cluster": "C", "article_file": "articles/agent-system-evaluation.md", "claim_type": "methodological", "location": "L33", "claim": "Trace grading assesses tool selection, handoffs and workflow behaviour", "source_url": "https://developers.openai.com/api/docs/guides/agent-evals", "source_title": "Evaluate agent workflows", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide trace-grading workflow enumerates those questions.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-002", "cluster": "C", "article_file": "articles/agent-system-evaluation.md", "claim_type": "empirical_result", "location": "L43", "claim": "2026 SWE-Bench Pro Verified reports solution/evaluation leakage and task-quality problems", "source_url": "https://arxiv.org/abs/2609.08149", "source_title": "SWE-Bench Pro Verified: A Reliable Benchmark for Software Engineering Agents", "source_date": "2026-09-08; revised 2026-10-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original preprint abstract identifies reward-hacking channels and misleading statements/scoped tests; article appropriately limits scope.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-003", "cluster": "C", "article_file": "articles/agent-system-evaluation.md", "claim_type": "methodological", "location": "L61", "claim": "Model/harness attribution requires controlled component changes", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Original paper distinguishes downstream-system evaluation from model evaluation; ablation is a reasoned design implication.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-004", "cluster": "C", "article_file": "articles/build-task-eval.md", "claim_type": "methodological", "location": "L33–35", "claim": "Oversampled risk/synthetic examples do not establish production prevalence", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official task-specific evaluation and production-pattern anti-pattern guidance support the sampling principle.", "limits": "No external citations occur in this article; this is reviewer-supplied support and a mathematical sampling check, not evidence the producer verified a specific data mix.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-005", "cluster": "C", "article_file": "articles/build-task-eval.md", "claim_type": "methodological", "location": "L43", "claim": "Independent labelling and adjudication expose rubric ambiguity but agreement does not prove correctness", "source_url": "https://arxiv.org/abs/2404.12272", "source_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "source_date": "2024-04-18", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Original abstract finds criteria drift and iterative human validation; the prescribed two-reviewer protocol is editorial.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-006", "cluster": "C", "article_file": "articles/build-task-eval.md", "claim_type": "methodological", "location": "L47,59–61", "claim": "Grouped cases and variation/decision margin affect independent sample needs", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Sections 2.2 and 5 establish clustered sampling and effect-size/variance dependencies.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-007", "cluster": "C", "article_file": "articles/choosing-model-benchmark.md", "claim_type": "substantive_factual", "location": "L33", "claim": "MMLU covers 57 subjects and HELM evaluates multiple desiderata", "source_url": "https://arxiv.org/abs/2009.03300", "source_title": "Measuring Massive Multitask Language Understanding", "source_date": "2020-09-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "MMLU original abstract confirms scope; HELM original abstract separately confirms named metrics.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-008", "cluster": "C", "article_file": "articles/choosing-model-benchmark.md", "claim_type": "changing_pricing_structure", "location": "L76", "claim": "Provider pricing includes multiple token classes and tool charges", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official token-category and Tools pricing sections show distinct charge bases.", "limits": "The draft reproduces no rates; current structure corroborated rather than replacing its 22 October observation.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-009", "cluster": "C", "article_file": "articles/choosing-model-benchmark.md", "claim_type": "substantive_factual", "location": "L84", "claim": "SWE-bench tests issue-driven codebase repair and depends on execution conditions", "source_url": "https://arxiv.org/abs/2310.06770", "source_title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "source_date": "2023-10-10", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original paper describes repository plus issue and executable assessment.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-010", "cluster": "C", "article_file": "articles/datasets-holdouts-contamination.md", "claim_type": "methodological", "location": "L21", "claim": "Random row splits can be misleading when records share documents/entities", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Cluster framework shows non-independent observations; partition-by-group guidance is a methodological inference.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-011", "cluster": "C", "article_file": "articles/datasets-holdouts-contamination.md", "claim_type": "methodological", "location": "L31 first citation", "claim": "Yang paper proposes comparing original/rephrased test performance as a memorisation probe", "source_url": "https://arxiv.org/abs/2311.04850", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_date": "2023-11-08", "accessed_at_utc": "2026-10-29", "verdict": "citation_method_mismatch", "supporting_evidence": "Full paper sections 4–5 train on rephrased test items and evaluate contamination detection in training corpora. This is not the same black-box original-versus-paraphrase score audit described in the draft.", "limits": "Underlying warning that exact matching misses semantic overlap is supported. Cite the actual audit method (e.g. the nearby 2026 paper) or describe Yang’s method faithfully; real paper, not fabricated evidence.", "producer_research_date": "2026-10-22" }, { "id": "C-012", "cluster": "C", "article_file": "articles/datasets-holdouts-contamination.md", "claim_type": "empirical_result", "location": "L31 second citation", "claim": "2026 contamination study reports score inflation without necessarily changing relative leaderboard order", "source_url": "https://arxiv.org/abs/2609.02899", "source_title": "Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards", "source_date": "2026-07-05 as shown by live arXiv record; identifier begins 2609", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports a paraphrase-controlled study on four benchmarks with rank correlation .997 and explains differential versus uniform contamination.", "limits": "Article wisely gives no new numeric result and labels scope/preprint. Abstract checked; no reanalysis or peer-review determination.", "producer_research_date": "2026-10-22" }, { "id": "C-013", "cluster": "C", "article_file": "articles/fair-model-comparisons.md", "claim_type": "methodological", "location": "L19–29", "claim": "A common harness helps attribution; different native interfaces imply system comparison", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Paper identifies conflation of model/downstream developer needs and lack of standardisation.", "limits": "No external citations in this article. Reviewer confirms logic; no primary study is claimed to validate this exact control plan.", "producer_research_date": "2026-10-22" }, { "id": "C-014", "cluster": "C", "article_file": "articles/fair-model-comparisons.md", "claim_type": "methodological", "location": "L33", "claim": "Tune on development/validation and reserve final holdout", "source_url": "https://arxiv.org/abs/2407.01502", "source_title": "AI Agents That Matter", "source_date": "2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Original abstract identifies inadequate holdouts and benchmark overfitting as problems.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-015", "cluster": "C", "article_file": "articles/fair-model-comparisons.md", "claim_type": "methodological", "location": "L35,63", "claim": "Pair cases and quantify uncertainty instead of interpreting tiny differences alone", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Sections 4–5 support paired inference and decision-relevant effect sizes.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-016", "cluster": "C", "article_file": "articles/human-evaluation-model-judges.md", "claim_type": "methodological", "location": "L35", "claim": "Position bias can change judge preferences when order swaps", "source_url": "https://arxiv.org/abs/2406.07791", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_date": "2024-06-12; revised 2025-11-11", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms position bias across judges/tasks; substance is supported, author label is wrong.", "limits": "Bibliographic author correction is recorded separately and not counted as fabricated evidence.", "producer_research_date": "2026-10-22" }, { "id": "C-017", "cluster": "C", "article_file": "articles/human-evaluation-model-judges.md", "claim_type": "methodological", "location": "L39", "claim": "Judges can prefer verbosity/self-like outputs", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original MT-bench abstract documents verbosity and self-enhancement biases.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-018", "cluster": "C", "article_file": "articles/human-evaluation-model-judges.md", "claim_type": "methodological", "location": "L43–51", "claim": "Use criteria/examples and human-calibrated model judges", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official tips and grader-design principles support feedback calibration and criteria-based evaluation.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-019", "cluster": "C", "article_file": "articles/latency-cost.md", "claim_type": "changing_pricing_structure", "location": "L29", "claim": "Official pricing separates token categories and some tool calls", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Pricing token table and Tools section distinguish input/output/cache and extra tool charges.", "limits": "No fixed rates are asserted; the source structure was checked live without claiming historical rates or archived retrieval.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-020", "cluster": "C", "article_file": "articles/latency-cost.md", "claim_type": "methodological", "location": "L39", "claim": "Means can conceal a slow latency tail; report percentiles/load", "source_url": "https://sre.google/sre-book/monitoring-distributed-systems/", "source_title": "Monitoring Distributed Systems — Google SRE", "source_date": "2016 book; live text", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary Google SRE chapter explains response-time tails and saturation-sensitive measurements.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-021", "cluster": "C", "article_file": "articles/latency-cost.md", "claim_type": "hypothetical_arithmetic", "location": "L57", "claim": "Hypothetical total costs $258 and $127 after reviewer time", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Arithmetic checked: 18+60*4*1=258; 27+25*4*1=127. If all corrected cases meet contract, denominator=500 and costs per completed case are .516/.254.", "limits": "Invented workloads/prices, not provider results. Pricing URL supports accounting unit structure only; arithmetic is reviewer calculation. Avoid double-counting retries already included in total input/output charges at L27.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-022", "cluster": "C", "article_file": "articles/maintaining-evaluations.md", "claim_type": "methodological", "location": "L25", "claim": "Continuous eval on changes and new observed cases is recommended", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official continuous-evaluation design step and task-specific tips agree.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-023", "cluster": "C", "article_file": "articles/maintaining-evaluations.md", "claim_type": "methodological", "location": "L38–40", "claim": "Changing suite composition can move the score independently of model changes", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Common-scenario design supports comparability; weighted-average/sample-composition logic confirms the guidance.", "limits": "Version bridges are editorial advice, not a protocol experimentally validated by the cited paper.", "producer_research_date": "2026-10-22" }, { "id": "C-024", "cluster": "C", "article_file": "articles/maintaining-evaluations.md", "claim_type": "methodological", "location": "L46", "claim": "Judge/rubric changes need human-labelled calibration before comparing scores", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Maintain-agreement principle supports recalibration; preserving original/retrospective scales is sensible editorial method.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-025", "cluster": "C", "article_file": "articles/metrics-rubrics.md", "claim_type": "methodological", "location": "L21", "claim": "Precision/recall differ and an always-majority classifier can miss all minority urgent cases", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html", "source_title": "precision_score — scikit-learn", "source_date": "live v1.9.1 documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official definitions match; hypothetical 95% majority arithmetic gives 95% accuracy and zero urgent recall.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-026", "cluster": "C", "article_file": "articles/metrics-rubrics.md", "claim_type": "methodological", "location": "L31", "claim": "Generic metrics alone can miss intended application behaviour", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official anti-pattern guidance warns against generic academic metrics and biased design.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "source_date_as_initially_catalogued": "2026", "producer_research_date": "2026-10-22" }, { "id": "C-027", "cluster": "C", "article_file": "articles/metrics-rubrics.md", "claim_type": "methodological", "location": "L45", "claim": "Graders include exact/similarity/code/model methods and should be validated against known-quality examples", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_title": "Graders", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official grader categories and How to write grader prompts section describe expert-labelled examples and known ordering.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-028", "cluster": "C", "article_file": "articles/metrics-rubrics.md", "claim_type": "methodological", "location": "L35", "claim": "Groundedness anchors 1 and 2 distinguish minor unsupported details versus all material claims supported", "source_url": "https://developers.openai.com/api/docs/guides/graders", "source_title": "Graders", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "ambiguous_methodological_guidance", "supporting_evidence": "Reviewer anchor check finds overlap: an answer with only a minor unsupported detail satisfies both level 1 and level 2’s all-material-supported definition. Official known-quality validation would expose this ambiguity.", "limits": "Ordinary substantive rubric correction: define level 2 as no unsupported claims, or otherwise make adjacent levels exclusive. No fabricated result.", "producer_research_date": "2026-10-22" }, { "id": "C-029", "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "claim_type": "substantive_factual", "location": "L21", "claim": "MMLU is a 57-subject broad academic/professional test", "source_url": "https://arxiv.org/abs/2009.03300", "source_title": "Measuring Massive Multitask Language Understanding", "source_date": "2020-09-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms scope.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-030", "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "claim_type": "substantive_factual", "location": "L31", "claim": "HELM reports multi-scenario/multi-metric standardised evaluation", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms the stated desiderata and bounded scope.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-031", "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "claim_type": "methodological", "location": "L39", "claim": "Swapping output order can affect LLM-judge preferences", "source_url": "https://arxiv.org/abs/2406.07791", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_date": "2024-06-12; revised 2025-11-11", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original study abstract supports position bias; author label requires separate bibliographic correction.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-032", "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "claim_type": "methodological", "location": "L45", "claim": "Yang rephrased-sample study is a direct original-versus-rewrite memorisation probe", "source_url": "https://arxiv.org/abs/2311.04850", "source_title": "Rethinking Benchmark and Contamination for Language Models with Rephrased Samples", "source_date": "2023-11-08", "accessed_at_utc": "2026-10-29", "verdict": "citation_method_mismatch", "supporting_evidence": "Full paper trains on rephrased test data and checks contamination detection; the draft’s method characterisation is imprecise.", "limits": "Correctly warns that paraphrase checks do not prove absence. Replace attribution or describe the contamination-training demonstration accurately.", "producer_research_date": "2026-10-22" }, { "id": "C-033", "cluster": "C", "article_file": "articles/repeatability-uncertainty.md", "claim_type": "methodological", "location": "L30", "claim": "Three repeats of 100 cases give 300 outputs but not 300 independent cases", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Section 3.1 warns pooled KN standard errors are inconsistent and retains question-level means.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-034", "cluster": "C", "article_file": "articles/repeatability-uncertainty.md", "claim_type": "methodological", "location": "L42", "claim": "Bootstrap independent units, preserving conversation/repository groups", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_title": "bootstrap — SciPy", "source_date": "live v1.18.0 documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official resampling procedure and paired shared-index option support unit-aware resampling; Miller 2.2 establishes why groups matter.", "limits": "Primary source text checked; not reproduced experimentally. Source date is distinct from review access date; local prescriptions remain editorial judgments.", "producer_research_date": "2026-10-22" }, { "id": "C-035", "cluster": "C", "article_file": "articles/repeatability-uncertainty.md", "claim_type": "methodological", "location": "L44", "claim": "Benchmark variance includes sampling/training/hyperparameter sources", "source_url": "https://arxiv.org/abs/2103.03098", "source_title": "Accounting for Variance in Machine Learning Benchmarks", "source_date": "2021-03-01", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_qualification", "supporting_evidence": "Original abstract explicitly models sources of variance and proposes comparison recommendations.", "limits": "Training-pipeline variance is not identical to hosted-generation variation; cluster bootstrap prescription should cite a closer method source. Abstract alone does not verify every confidence-interval/test detail.", "producer_research_date": "2026-10-22" }, { "id": "D-001", "cluster": "D", "article_file": "articles/ai-latency-cost-per-successful-task.md", "claim_type": "methodological", "location": "L21", "claim": "Streaming reduces waiting for visible output without defining task completion", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_title": "Latency optimization", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official latency guide make-users-wait-less section; draft carefully separates perceived timing from completed task.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-002", "cluster": "D", "article_file": "articles/ai-latency-cost-per-successful-task.md", "claim_type": "changing_pricing_structure", "location": "L41–43", "claim": "Token/cache/tool categories need category-aware accounting; avoid adding included reasoning twice", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official pricing tables/Tools section show distinct units; official reasoning guide confirms reasoning billed in output.", "limits": "No dated numeric supplier rates asserted; live category structure corroborates the 22 October observation without an archive.", "producer_research_date": "2026-10-22" }, { "id": "D-003", "cluster": "D", "article_file": "articles/ai-latency-cost-per-successful-task.md", "claim_type": "changing_platform_specification", "location": "L51", "claim": "Cache reuse depends on full rendered prefix plus model-dependent eligibility/lookup", "source_url": "https://developers.openai.com/api/docs/guides/prompt-caching", "source_title": "Prompt caching", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official guide says entire rendered prefix must match and lists breakpoint/lookup and minimum-length differences.", "limits": "Current documentation checked; implementation details can change and no historical archive was inspected. No cache hit or savings experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-004", "cluster": "D", "article_file": "articles/ai-latency-cost-per-successful-task.md", "claim_type": "hypothetical_arithmetic", "location": "L59–61", "claim": "Hypothetical costs .008 per attempt and .015/.0167 per accepted task", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Recomputed (2000*2+500*8)/1e6=.008; 1.2/80=.015; 1.5/90=.0166666667.", "limits": "Invented rates and outcomes; pricing URL supports units only, not the invented prices or performance. Arithmetic was independently calculated.", "producer_research_date": "2026-10-22" }, { "id": "D-005", "cluster": "D", "article_file": "articles/ai-model-benchmarking-guide.md", "claim_type": "substantive_factual", "location": "L33", "claim": "Original MMLU spans 57 subjects including mathematics/history/law", "source_url": "https://arxiv.org/abs/2009.03300", "source_title": "Measuring Massive Multitask Language Understanding", "source_date": "2020-09-07; revised 2021-01-12", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms task breadth and broad knowledge/problem-solving objective.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-006", "cluster": "D", "article_file": "articles/ai-model-benchmarking-guide.md", "claim_type": "methodological", "location": "L73", "claim": "Task-specific evaluations and human-calibrated automated scoring are official recommendations", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official tips and workflow guidance explicitly state both.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-007", "cluster": "D", "article_file": "articles/ai-model-benchmarking-guide.md", "claim_type": "hypothetical_arithmetic", "location": "L105", "claim": "Hypothetical $12 over 80 accepted outcomes yields $.15", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "Pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Direct recalculation 12/80=.15; 100 attempts minus 80 accepted leaves 20 failures.", "limits": "Calculated illustration, not provider rates or model results; official pricing reference supplies accounting context only.", "producer_research_date": "2026-10-22" }, { "id": "D-008", "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "claim_type": "hypothetical_arithmetic", "location": "L31–33", "claim": "Hypothetical joint table yields A80/B85, lead5pp, 15 discordances", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Recomputed 75+10+5+10=100; A=75+5; B=75+10; discordant=5+10; source establishes paired-analysis relevance.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-009", "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "claim_type": "hypothetical_arithmetic", "location": "L39–43", "claim": "80/100 Wilson95 interval approximately71.1–86.7, with frequentist coverage interpretation", "source_url": "https://www.itl.nist.gov/div898/handbook/prc/section2/prc241.htm", "source_title": "7.2.4.1. Confidence intervals", "source_date": "publication identity in source-retrieval-19/20/21.json", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "NIST Wilson formula recalculated at z=1.96: [.7111690380734977,.8666340774409013]; SciPy bootstrap introduction separately supports coverage interpretation.", "limits": "Independent-binomial illustration only; not actual candidate success probability or an assertion that every future output falls in the interval.", "producer_research_date": "2026-10-22" }, { "id": "D-010", "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "claim_type": "methodological", "location": "L49–55", "claim": "Paired resampling preserves candidate pairs; percentile tails2.5/97.5 and small/degenerate limits", "source_url": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html", "source_title": "scipy.stats.bootstrap", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official shared-index paired option and bootstrap method descriptions agree. Group resampling is explicitly an extension with target weighting specified.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-011", "cluster": "D", "article_file": "articles/design-llm-evaluation-tasks.md", "claim_type": "substantive_factual", "location": "L55", "claim": "CheckList uses linguistic-capability/test-type matrix for behavioural tests beyond held-out accuracy", "source_url": "https://aclanthology.org/2020.acl-main.442/", "source_title": "Beyond Accuracy: Behavioral Testing of NLP Models with CheckList", "source_date": "2020-07", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original ACL abstract states this design; draft does not generalise user-study effect sizes.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-012", "cluster": "D", "article_file": "articles/design-llm-evaluation-tasks.md", "claim_type": "methodological", "location": "L57", "claim": "Representative and designed challenge samples answer different population questions", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official production-distribution guidance supports this distinction; weighting follows sampling logic.", "limits": "Editorial sampling method, not a measured study of RETURN-014; no production prevalence is asserted.", "producer_research_date": "2026-10-22" }, { "id": "D-013", "cluster": "D", "article_file": "articles/design-llm-evaluation-tasks.md", "claim_type": "methodological", "location": "L65–75", "claim": "Near-duplicate variants should remain grouped; sample needs depend on independent groups/variance/decision margin", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Clustered-unit and power-analysis framework confirms those dependencies.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-014", "cluster": "D", "article_file": "articles/evaluation-dataset-leakage-holdouts.md", "claim_type": "methodological", "location": "L21", "claim": "Leakage uses unavailable-at-prediction information; split before fitting transforms", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_title": "Common pitfalls and recommended practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official common-pitfalls Data leakage section gives definition and split-before-preprocessing advice.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-015", "cluster": "D", "article_file": "articles/evaluation-dataset-leakage-holdouts.md", "claim_type": "substantive_factual", "location": "L23", "claim": "Pretraining detection studies black-box access without training-corpus knowledge", "source_url": "https://arxiv.org/abs/2310.16789", "source_title": "Detecting Pretraining Data from Large Language Models", "source_date": "2023-10-25; revised 2024-03-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract defines this setting and Min-K% Prob; draft correctly does not claim universal certification.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-016", "cluster": "D", "article_file": "articles/evaluation-dataset-leakage-holdouts.md", "claim_type": "substantive_factual", "location": "L31", "claim": "Datasheets documents motivation/composition/collection/intended uses", "source_url": "https://arxiv.org/abs/1803.09010", "source_title": "Datasheets for Datasets", "source_date": "2018-03-23; revised 2021-12-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract proposes those documentation elements; compact local adaptation is labelled.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-017", "cluster": "D", "article_file": "articles/evaluation-dataset-leakage-holdouts.md", "claim_type": "substantive_factual", "location": "L65", "claim": "Canonical-versus-shuffled likelihood test depends on exchangeability", "source_url": "https://arxiv.org/abs/2310.17623", "source_title": "Proving Test Set Contamination in Black Box Language Models", "source_date": "2023-10-26; revised 2023-11-24", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract explicitly states exchangeable benchmark/order-likelihood mechanism and black-box setting.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-018", "cluster": "D", "article_file": "articles/fair-ai-model-comparisons.md", "claim_type": "empirical_result", "location": "L33", "claim": "Meaning-preserving prompt formats affected studied few-shot results and transferred weakly across models", "source_url": "https://arxiv.org/abs/2310.11324", "source_title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting", "source_date": "2023-10-17; revised 2024-07-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract reports sensitivity and weak correlation; draft carefully omits universal magnitude.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-019", "cluster": "D", "article_file": "articles/fair-ai-model-comparisons.md", "claim_type": "changing_model_specification", "location": "L51", "claim": "OpenAI reasoning tokens use context and are billed as output while not normal visible text", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official accounting passage confirms; adaptive/model-dependent controls support avoiding internal-compute equivalence.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-020", "cluster": "D", "article_file": "articles/fair-ai-model-comparisons.md", "claim_type": "changing_software_capability", "location": "L75", "claim": "EleutherAI harness is a shared framework for language-model evaluation", "source_url": "https://github.com/EleutherAI/lm-evaluation-harness", "source_title": "Language Model Evaluation Harness", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer repository description and README expose tasks/model arguments; no fairness guarantee is claimed.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-021", "cluster": "D", "article_file": "articles/human-reviewers-llm-judges.md", "claim_type": "substantive_factual", "location": "L49", "claim": "MT-bench documents position/verbosity/self-enhancement bias and reasoning limits", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09; revised 2023-12-24", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract names these; draft limits agreement to study conditions.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-022", "cluster": "D", "article_file": "articles/human-reviewers-llm-judges.md", "claim_type": "methodological", "location": "L33,43", "claim": "Fresh independent human-labelled cases and structured criteria are needed for judge validation", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official human calibration/evaluation design supports process; producer clearly labels the exact workflow editorial.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-023", "cluster": "D", "article_file": "articles/human-reviewers-llm-judges.md", "claim_type": "hypothetical_arithmetic", "location": "L61", "claim": "Hypothetical always-pass judge agrees90% when90 of100 pass yet detects no failures", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "2023-06-09; revised 2023-12-24", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Direct confusion-matrix check: TP90, FP10, TN0, FN0; failure-detection recall0/10 and agreement90/100.", "limits": "Arithmetic counterexample only; cited paper supplies bias context, not this invented judge result.", "producer_research_date": "2026-10-22" }, { "id": "D-024", "cluster": "D", "article_file": "articles/llm-evaluation-metrics-rubrics.md", "claim_type": "hypothetical_arithmetic", "location": "L39–43", "claim": "Hypothetical TP15 FP10 FN5 TN70 yields precision60, recall75, F1≈66.7 and accuracy85", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_title": "precision_recall_fscore_support", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official definitions applied:15/25=.6;15/20=.75;30/(30+10+5)=2/3;(15+70)/100=.85; always-negative80/100=.8.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-025", "cluster": "D", "article_file": "articles/llm-evaluation-metrics-rubrics.md", "claim_type": "methodological", "location": "L47", "claim": "Macro gives classes equal weight; traffic-weighted summaries answer another question", "source_url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html", "source_title": "precision_recall_fscore_support", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official macro/weighted parameter definitions distinguish unweighted class means and support-weighted means.", "limits": "Traffic interpretation requires a representative sampling distribution; article correctly asks to explain averaging/counts.", "producer_research_date": "2026-10-22" }, { "id": "D-026", "cluster": "D", "article_file": "articles/llm-evaluation-metrics-rubrics.md", "claim_type": "methodological", "location": "L51–67", "claim": "Anchored task gates must separate material errors, omissions and valid answers from style preferences", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official criteria/examples and task-specific scoring guidance support the method; anchors examined for practical exclusivity.", "limits": "The exact0/1/2 support-policy rubric is an editorial design and not an experimentally validated scoring instrument. Some boundary examples could be added in publication editing.", "producer_research_date": "2026-10-22" }, { "id": "D-027", "cluster": "D", "article_file": "articles/maintain-llm-evaluation-suite.md", "claim_type": "methodological", "location": "L23", "claim": "Continuous evaluation expands cases based on monitoring/changes", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official workflow step5 and tips explicitly recommend ongoing evaluation/new cases.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-028", "cluster": "D", "article_file": "articles/maintain-llm-evaluation-suite.md", "claim_type": "methodological", "location": "L31–35,49–51", "claim": "Changed task/reference versions must be separated from changed candidate performance", "source_url": "https://scikit-learn.org/stable/common_pitfalls.html", "source_title": "Common pitfalls and recommended practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Leakage/timing and held-out evaluation principles support controlling the measurement boundary; same-output/different-score diagnosis is logical.", "limits": "Local version/triage policy is editorial method, not a primary-source empirical result.", "producer_research_date": "2026-10-22" }, { "id": "D-029", "cluster": "D", "article_file": "articles/maintain-llm-evaluation-suite.md", "claim_type": "methodological", "location": "L61–63", "claim": "Incident-derived variants are grouped diagnostic evidence rather than independent traffic samples", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Clustered-observation framework confirms shared-origin dependence; failure-selected sampling does not estimate prevalence.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-030", "cluster": "D", "article_file": "articles/read-ai-benchmark-leaderboards.md", "claim_type": "substantive_factual", "location": "L23,35", "claim": "HumanEval tests docstring-conditioned functional correctness; any-of-k differs from first attempt", "source_url": "https://arxiv.org/abs/2107.03374", "source_title": "Evaluating Large Language Models Trained on Code", "source_date": "2021-07-07; v2 2021-07-14", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original full paper section2.1 and HumanEval description support task and sampling interpretation.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-031", "cluster": "D", "article_file": "articles/read-ai-benchmark-leaderboards.md", "claim_type": "changing_benchmark_specification", "location": "L43", "claim": "Maintainer Verified/Bash Only view uses mini-SWE-agent and distinguishes variants", "source_url": "https://www.swebench.com/", "source_title": "SWE-bench Leaderboards", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live maintainer explanatory sections match the dated view description.", "limits": "29 October corroboration, not an independent archived22 October observation; no ranks/results inferred.", "producer_research_date": "2026-10-22" }, { "id": "D-032", "cluster": "D", "article_file": "articles/read-ai-benchmark-leaderboards.md", "claim_type": "substantive_factual", "location": "L53", "claim": "HELM compares multiple scenarios/metrics beyond accuracy", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "2022-11-16; revised 2023-10-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms the bounded design example.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-033", "cluster": "D", "article_file": "articles/evaluate-ai-agents-tools-outcomes.md", "claim_type": "methodological", "location": "L43–47", "claim": "tau-bench uses annotated final state; correct final state need not establish required confirmation/policy", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Abstract and full section3 explicitly distinguish goal-state reward from sufficient policy adherence.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "D-034", "cluster": "D", "article_file": "articles/evaluate-ai-agents-tools-outcomes.md", "claim_type": "methodological", "location": "L63", "claim": "Lost write acknowledgement requires contract-aware reconciliation/idempotent recovery", "source_url": "https://aws.amazon.com/builders-library/making-retries-safe-with-idempotent-APIs/", "source_title": "Making retries safe with idempotent APIs", "source_date": "undated live primary article", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary AWS engineering article Retrying and side effects describes no-response ambiguity, duplicate-write risk and reconciliation/client request IDs.", "limits": "Reviewer-supplied source corroborates editorial guidance; the local hypothetical return tool was not executed or proven safe.", "producer_research_date": "2026-10-22" }, { "id": "D-035", "cluster": "D", "article_file": "articles/evaluate-ai-agents-tools-outcomes.md", "claim_type": "methodological", "location": "L73–75", "claim": "pass^k averages all-k-success probability across tasks; pass@k any-success; power of mean is not generally correct", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Full section3 gives task-averaged binomial-combination estimators. Heterogeneous-task example p=0/1 with equal weights: E[p²]=.5 versus E[p]²=.25; draft warning is sound.", "limits": "Primary text checked and reasoning/arithmetic inspected where indicated; no empirical study, model evaluation or live service integration was reproduced.", "producer_research_date": "2026-10-22" }, { "id": "A-058", "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "claim_type": "empirical_result", "location": "L76", "claim": "Retail pass^8<25% is paired with single-attempt success under50%", "source_url": "https://arxiv.org/html/2406.12045v1#S5", "source_title": "tau-bench — original full paper", "source_date": "2024-06-17", "accessed_at_utc": "2026-10-29", "verdict": "misleading_scope", "supporting_evidence": "Section5/table2 and consistency discussion give retail gpt-4o pass1=61.2% and pass8<25%; overall domain average is48.2%. The abstract under50% and retail reliability result must not be read as the same retail single-attempt operating point.", "limits": "Historical result-scope correction; genuine primary source, no fabrication or experimental reproduction alleged.", "producer_research_date": "2026-10-25" }, { "id": "E-001", "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "claim_type": "substantive_factual", "location": "L21", "claim": "tau-bench checks final database state, rather than the agent’s closing sentence", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "ICLR 2025", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original paper section3 supports the task outcome distinction; scope is that benchmark’s reward.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-002", "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "claim_type": "substantive_factual", "location": "L23", "claim": "BFCL AST evaluation matches function names and allowed parameter values", "source_url": "https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html", "source_title": "Berkeley Function Calling Leaderboard", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer sections Function Matching and Required Parameters Matching agree; AST is one evaluation mode, not all BFCL tasks.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-003", "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "claim_type": "substantive_factual", "location": "L25", "claim": "BERTScore uses contextual-embedding token similarity", "source_url": "https://arxiv.org/abs/1904.09675", "source_title": "BERTScore: Evaluating Text Generation with BERT", "source_date": "2019", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms the mechanism; the draft’s refusal to treat resemblance as sufficient grounding is editorial judgement.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-004", "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "claim_type": "substantive_factual", "location": "L25", "claim": "G-Eval discusses low conventional-metric correlation on diverse outputs", "source_url": "https://arxiv.org/abs/2303.16634", "source_title": "G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment", "source_date": "2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract supports the limited correlation statement; no correlation number is invented.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-005", "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "claim_type": "methodological", "location": "L33–36", "claim": "Source occurrence can be treated as the complete automatic grounding rule", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "underspecified_guidance", "supporting_evidence": "Counterexample: source rent $2,000 and bond $8,000; a draft swapping them contains only source numbers but is ungrounded. Official task-specific/human-calibrated evaluation principles do not validate a presence-only checker.", "limits": "Reviewer-designed counterexample, not a source experiment. Structured fact/relationship validation could implement grounding; token/number occurrence alone cannot. Needs semantic validation examples, not a claim that code can never grade facts.", "producer_research_date": "2026-10-29" }, { "id": "E-006", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "empirical_result", "location": "L29", "claim": "MMLU-Pro prompt sweep:24 styles, MMLU4–5/max10.98 versus Pro≈2/max3.74", "source_url": "https://arxiv.org/abs/2406.01574", "source_title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "source_date": "NeurIPS 2024; arXiv v6, 6 November 2024", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Full v6 section6.3 contains those scoped values.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-007", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "changing_model_specification", "location": "L35", "claim": "Reasoning consumes output cap and is billed even without visible answer", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official reasoning-token/cap passages match. Nominal effort across different model families still needs qualification.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-008", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "changing_pricing", "location": "L37", "claim": "Fast/Ultrafast Astra rates versus standard; Opus fast2x price/up to2.5x speed", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live pricing standard/Fast/Ultrafast tables show10/50,20/100,60/300. Claude platform L169–171 independently confirms Opus vendor claim.", "limits": "Both providers actually retrieved29 October; price and advertised speed verified, no evidence that queue tier itself changes intrinsic answer quality. No speed experiment.", "producer_research_date": "2026-10-29" }, { "id": "E-009", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "changing_pricing", "location": "L39", "claim": "Residency uplifts:Anthropic1.1x/OpenAI10% from5March2026", "source_url": "https://claude.com/platform/api", "source_title": "Claude Platform", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live platform L169 and OpenAI pricing note L1000 match the qualified vendor statements.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-010", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "changing_benchmark_specification", "location": "L43", "claim": "mini-SWE-agent1.x/2.x results not necessarily comparable", "source_url": "https://www.swebench.com/verified.html", "source_title": "SWE-bench Verified", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Maintainer release note distinguishes text-parsed actions from tool calling.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-011", "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "claim_type": "changing_vendor_performance_claim", "location": "L47", "claim": "Vendor advertises caching up to97% cost and≈60% latency reductions", "source_url": "https://claude.com/platform/api", "source_title": "Claude Platform", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live platform L362–364 supports the attributed ceiling.", "limits": "Vendor claim only; no cache experiment or claim those gains apply to this fictional workload.", "producer_research_date": "2026-10-29" }, { "id": "E-012", "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "claim_type": "methodological", "location": "L19", "claim": "Anthropic recommends starting with20–50 real-failure tasks for large early effects", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Roadmap step0 supports the conditional recommendation, not universal statistical adequacy.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-013", "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "claim_type": "changing_documentation_recommendation", "location": "L21", "claim": "Official guide names Astra synthetic edge-case generation with human labellers", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live guide L1049–1053 confirms the dated documentation wording.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-014", "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "claim_type": "methodological", "location": "L36–38,67", "claim": "Reference solution, unstated filepath failure and pass@100 warning are Anthropic guidance", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Roadmap step2 explicitly gives those bounded examples/recommendations.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-015", "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "claim_type": "methodological", "location": "L59", "claim": "Remove a failed holdout item once it drives prompt tuning and exclude it from the reported score", "source_url": "https://scikit-learn.org/stable/modules/cross_validation.html", "source_title": "Cross-validation: evaluating estimator performance", "source_date": "live 1.9.1 documentation", "accessed_at_utc": "2026-10-29", "verdict": "materially_incorrect_guidance", "supporting_evidence": "Official guidance requires final testing independent of tuning. Removing failure-selected cases changes the test distribution:180/200 becomes180/199 without a better candidate.", "limits": "Reviewer counterexample illustrates selection bias. Retire the exposed evaluation or use a fresh independent holdout; moving a disclosed case into development does not establish that the remaining selected subset is unbiased.", "producer_research_date": "2026-10-29" }, { "id": "E-016", "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "claim_type": "changing_documentation_example", "location": "L61", "claim": "1000 transcripts/ROUGE-L≥.40/G-Eval coherence≥80% are documentation targets", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live example L1024–1034 states these design targets; draft correctly does not call them observed results.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-017", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "empirical_result", "location": "L19", "claim": "HELM historical coverage17.9→96%,30 models and seven metrics", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "TMLR 2023; arXiv revised 1 October 2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract checked earlier confirms those historical design/results quantities.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-018", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "changing_software_status", "location": "L19", "claim": "HELM maintenance mode began1June2026", "source_url": "https://crfm-helm.readthedocs.io/", "source_title": "HELM documentation, maintenance-mode notice", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live maintainer homepage L7 confirms date.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-019", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "changing_documentation_example", "location": "L33", "claim": "1000/.40/80% is an illustrative official design sketch", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live guide L1024–1034 matches; draft explicitly separates example from measurement.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-020", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "changing_pricing", "location": "L41", "claim": "Standard short-context Luna.10/.50,Sol2/10,Astra10/50 per million", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live pricing tables and fetched official markdown agree; reasoning guide supports inclusion of reasoning in output.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-021", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "changing_benchmark_audit_result", "location": "L88", "claim": "February2026 OpenAI audit reports flawed tests and contamination", "source_url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/", "source_title": "Why we no longer evaluate SWE-bench Verified", "source_date": "2026-02-23", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original live audit supports the bounded claim; no inference that all tasks are flawed.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-022", "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "claim_type": "changing_platform_status", "location": "L92", "claim": "Evals read-only31October/shutdown30November2026", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live dated Evals row L1066–1077 confirms scheduled dates.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-023", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "changing_benchmark_specification", "location": "L23", "claim": "Reference actions derive goal state and need not be followed unless reward demands them", "source_url": "https://github.com/sierra-research/tau2-bench", "source_title": "τ²-bench repository readme and domain notes", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Linked primary docs/evaluation.md L175–198 confirms reward_basis and DB versus ACTION distinction.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-024", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "changing_benchmark_specification", "location": "L25", "claim": "English2.0 not input-identical1.0.1; voice simulator gpt5.5 versus4.1", "source_url": "https://github.com/sierra-research/tau2-bench", "source_title": "τ²-bench repository readme and domain notes", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live README L256–266 confirms both compatibility and default changes.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-025", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "empirical_result", "location": "L25", "claim": "GPT4o under half is a paper-scoped aggregate, not current tau2 result", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "ICLR 2025", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_scope_clarification", "supporting_evidence": "Original full result table gives retail61.2/airline35.2, mean48.2. Draft does not explicitly call under-half retail, but should state the cross-domain aggregate.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-026", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "vendor_reported_incident", "location": "L45", "claim": "Shared git history advantaged agents in Anthropic’s internal eval account", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary post L306–307 reports this; draft clearly attributes the incident.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-027", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "changing_documentation_capability", "location": "L49", "claim": "Long-running-work page gives outcome/constraints/verification, strictTS example and goal command surfaces", "source_url": "https://learn.chatgpt.com/docs/long-running-work", "source_title": "Long-running work", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live page L980–997 confirms all; document citation is examined as evidence, not instructions to this reviewer.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-028", "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "claim_type": "vendor_reported_incident", "location": "L53", "claim": "Anthropic reports Opus4.5 booking loophole and METR threshold-grader issue", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary post L59 and L317 supports attribution. It is not independently reproduced incident evidence.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-029", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "methodological", "location": "L31", "claim": "LiveCodeBench uses dated LeetCode/AtCoder/Codeforces tasks", "source_url": "https://arxiv.org/abs/2403.07974", "source_title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "source_date": "2024", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original v1 design/contamination sections checked in earlier retrievals support time-sliced collection.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-030", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "changing_benchmark_audit_result", "location": "L33", "claim": "Verified500; audited27.6% subset had≥59.4% problematic tests", "source_url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/", "source_title": "Why we no longer evaluate SWE-bench Verified", "source_date": "2026-02-23", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live audit explicitly reports these quantities for a failure-selected sample; draft preserves the sample restriction.", "limits": "Source headline and broader body definition differ slightly; draft follows the headline, as its handoff discloses. Not extrapolated to whole benchmark.", "producer_research_date": "2026-10-29" }, { "id": "E-031", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "changing_benchmark_audit_result", "location": "L35", "claim": "JulyPro audit200/27.4% and249/34.1%; earlier recommendation retracted", "source_url": "https://openai.com/index/separating-signal-from-noise-in-coding-evaluations/", "source_title": "Separating signal from noise in coding evaluations", "source_date": "2026-07-08", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary audit L40 and L74 confirms counts and retraction.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-032", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "changing_review_conclusion", "location": "L35", "claim": "Epoch September review concludes static exposure/scoring concerns and Flawed verdict", "source_url": "https://epoch.ai/benchmarks/swe-bench-verified/review", "source_title": "SWE-bench Verified benchmark review", "source_date": "2026-09-03", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Epoch’s own review L135–166 supports attribution; OpenAI audit checked directly for underlying data.", "limits": "Epoch is primary for its own conclusion, secondary for the audit. No extra empirical check is counted for its recitation of OpenAI data.", "producer_research_date": "2026-10-29" }, { "id": "E-033", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "empirical_result", "location": "L37", "claim": "Code overlap paper finds substantial training overlap and higher scores on matched solutions", "source_url": "https://arxiv.org/abs/2403.04811", "source_title": "Quantifying contamination in evaluating code generation capabilities of language models", "source_date": "2024", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract states this direction; no exact effect size is asserted.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-034", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "substantive_factual", "location": "L41", "claim": "GAIA held out300 answers from466 questions", "source_url": "https://arxiv.org/abs/2311.12983", "source_title": "GAIA: a benchmark for General AI Assistants", "source_date": "2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original GAIA abstract checked in retrieval5 confirms both counts.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-035", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "methodological", "location": "L49", "claim": "After tuning on a failure, report score on remaining holdout and disclose shrinkage", "source_url": "https://scikit-learn.org/stable/modules/cross_validation.html", "source_title": "Cross-validation: evaluating estimator performance", "source_date": "live 1.9.1 documentation", "accessed_at_utc": "2026-10-29", "verdict": "materially_incorrect_guidance", "supporting_evidence": "Discarding hard exposed cases selects the reported sample by observed outcomes; disclosure of n does not remove bias. The official final-test/validation boundary and180/200→180/199 counterexample identify the defect.", "limits": "Selection-bias deduction, not an experiment. A fresh independent test or full labelled historical result is needed; cannot certify the remaining subset by moving the failed case.", "producer_research_date": "2026-10-29" }, { "id": "E-036", "cluster": "E", "article_file": "articles/human-review-and-model-judges.md", "claim_type": "empirical_result", "location": "L27", "claim": "MT-Bench GPT4 agreement>80% with scoped human comparisons and named biases", "source_url": "https://arxiv.org/abs/2306.05685", "source_title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "source_date": "NeurIPS 2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract and mitigation section support the study-scoped agreement/bias summary.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-037", "cluster": "E", "article_file": "articles/human-review-and-model-judges.md", "claim_type": "empirical_result", "location": "L27", "claim": "G-Eval identifies potential preference for LLM-written text", "source_url": "https://arxiv.org/abs/2303.16634", "source_title": "G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment", "source_date": "2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original abstract confirms this potential bias without generalising it to every family.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-038", "cluster": "E", "article_file": "articles/human-review-and-model-judges.md", "claim_type": "changing_documentation_recommendation", "location": "L29", "claim": "Official current guide recommends Astra then human agreement validation", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live L1174–1188 supports the attributed recommendation and caveat.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-039", "cluster": "E", "article_file": "articles/human-review-and-model-judges.md", "claim_type": "vendor_reported_incident", "location": "L31,49", "claim": "Unknown option, isolated dimensions and42→95CORE-Bench account attributed to Anthropic", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary post L313–316 confirms guidance and incident account.", "limits": "Vendor narrative, not independently verified CORE-Bench trial; draft accurately attributes rather than presenting its own measurement.", "producer_research_date": "2026-10-29" }, { "id": "E-040", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_api_specification", "location": "L17", "claim": "Optional shutdown_date on list/retrieve model endpoints", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live overview L969 confirms field,format,null and endpoints.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-041", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_platform_policy", "location": "L21", "claim": "Notice policy6monthsGA/3specialised/shortpreview plus exception", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live policy L974–978 matches qualified statement.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-042", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_model_availability", "location": "L25", "claim": "GPT4 listedsnapshots notified14Oct2026,shutdown14Jan2027,fine-tunes same,Luna replacement", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live table L990–1000 confirms all five snapshots and fine-tune schedule; draft honestly notes policy/row interval mismatch.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-043", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_model_availability", "location": "L27", "claim": "Codex5.3/5.1/nano5.4 removal1Apr2027 and replacements", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live table L1001–1009 matches dates,snapshots and replacements.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-044", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_model_availability", "location": "L29", "claim": "GPT5/o3families removal11Dec2026 with5.6 replacements/pro mode", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live table L1045–1055 supports stated list and replacement family.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-045", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_platform_status", "location": "L31", "claim": "EvalsJune3announcement/Oct31readonly/Nov30shutdown andPromptfoo pointer", "source_url": "https://developers.openai.com/api/docs/deprecations", "source_title": "Deprecations", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live Evals row L1066–1077 confirms dates and linked title.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-046", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_benchmark_status", "location": "L33,37", "claim": "HELMJune1maintenance; JulyPro recommendation retracted", "source_url": "https://crfm-helm.readthedocs.io/", "source_title": "HELM documentation, maintenance-mode notice", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "HELM primaryhomepageL7; OpenAI July originalauditL74 independently confirms retraction.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-047", "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "claim_type": "changing_pricing", "location": "L41", "claim": "ArgonDecember1ratechange and5.6SolpromotionatleastNov21", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "source_title": "Gemini API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live Argon row L131–147; official OpenAI pricing note L1000 confirms second date.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-048", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_pricing", "location": "L19–27", "claim": "OpenAIcontextthreshold and standard/cache/B.Flex/Fast/Ultrafast rates", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Fetched official markdown results/openai-pricing-live.md lines33–40,89–96,133–140,171–178,210–215 confirms≤272K/>272K,Astra20/2/75long and modes. Rendered live table independently checks short rates.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-049", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_pricing", "location": "L27", "claim": "Regional10%fromMarch5 and5.6Sol4/20promotionatleastNov21", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live pricing note and cyber/standard rows confirm dated values.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-050", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_api_accounting", "location": "L29", "claim": "Reasoning included inoutput_tokens,with detailbreakdown,billed evenwithoutvisibleanswer", "source_url": "https://developers.openai.com/api/docs/guides/reasoning", "source_title": "Reasoning models", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Official reasoning guide accounting/max-output passages checked in retrieval10/11.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-051", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_pricing", "location": "L31,47", "claim": "Fable/Opus/Sonnet/Haiku rates/cache,100Kbreak,batch50%,US1.1x,fast2x", "source_url": "https://claude.com/platform/api", "source_title": "Claude Platform", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live platform L51–173 confirms every stated rate/tier/multiplier; Fable write12.50 exceeds input10. Five-minute cache-write TTL should be stated.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-052", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_pricing", "location": "L33,43", "claim": "Argon2/10throughNov30then4/20,thinkingincluded,batchhalf,implicitcacheonly", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "source_title": "Gemini API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live Argon standard/batch rows L127–150 match.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-053", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "changing_pricing", "location": "L35", "claim": "Websearch10/1000pluscontent;containersmemory/sessionpricing", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live Tools table L1113–1126 confirms units; eightsearches fee8*.01=.08beforecontent.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-054", "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "claim_type": "hypothetical_arithmetic", "location": "L39–45,51", "claim": "Hypothetical call costs,retry increment,80koutput andcostpersuccess", "source_url": "https://developers.openai.com/api/docs/pricing", "source_title": "OpenAI API pricing", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_minor_arithmetic_wording_error", "supporting_evidence": "Recomputed output920; costs(.00066,.0132,.066);1000x(.66,13.2,66);retry .66*.3=.198;80k*50/1e6=4;.00066/.88=.00075;.066/.91≈.072527. Astra/Luna base ratio is100, not the stated fifty.", "limits": "Rates verified, token mix and pass rates explicitly invented. Replace fifty-fold with100-fold; conclusion remains valid. No invoices or real performance measured.", "producer_research_date": "2026-10-29" }, { "id": "E-055", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "methodological", "location": "L17,21", "claim": "Approximate1.96SEM normal interval and clusteredstandarderrors", "source_url": "https://www.anthropic.com/research/statistical-approach-to-model-evals", "source_title": "A statistical approach to model evaluations", "source_date": "2024-11-19", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary explainer recommendations1/2 support conditions and clustering. A normal interval is an approximation, not an exact universal confidence guarantee.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-056", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "hypothetical_arithmetic", "location": "L19,41–42", "claim": "Hypothetical binomial SE/interval arithmetic", "source_url": "https://arxiv.org/abs/2411.00640", "source_title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Recomputed180/200SE=.021213,95%[.85842,.94158];182interval[.87034,.94966];176interval[.83496,.92504]. Table rounds appropriately.", "limits": "Independent binary illustrative Wald intervals; no observed outcomes or guarantee of coverage at small/extreme rates.", "producer_research_date": "2026-10-29" }, { "id": "E-057", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "methodological", "location": "L19", "claim": "A rivalat.87 is not a demonstrated loss versus.90because inside.86–.94", "source_url": "https://www.anthropic.com/research/statistical-approach-to-model-evals", "source_title": "A statistical approach to model evaluations", "source_date": "2024-11-19", "accessed_at_utc": "2026-10-29", "verdict": "materially_incorrect_guidance", "supporting_evidence": "Primary paired-difference advice requires joint data. Counterexample on200items:Aonly6,Bonly0,both174,neither20 gives.90/.87 and exact two-sided paired sign/binomial p=2*(.5**6)=.03125.", "limits": "Illustrative mathematical counterexample, not an evaluation run. The later L44 correctly says marginals cannot decide paired significance, contradicting L19’s categorical inference. Correct before publication.", "producer_research_date": "2026-10-29" }, { "id": "E-058", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "methodological", "location": "L23,44", "claim": "Use paired differences; marginalratesalonecannotdeterminepairedinterval", "source_url": "https://www.anthropic.com/research/statistical-approach-to-model-evals", "source_title": "A statistical approach to model evaluations", "source_date": "2024-11-19", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Primary recommendation4 supports this distinction. It is the correct later advice, but does not cure L19.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-059", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "methodological", "location": "L29", "claim": "pass@kany/pass^kall; .75cubed=.421875", "source_url": "https://arxiv.org/abs/2107.03374", "source_title": "Evaluating Large Language Models Trained on Code", "source_date": "2021", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Original HumanEval estimator plus originaltau papersection3 confirm metrics; independent arithmetic verifies k3illustration.", "limits": "Uniform independent trial illustration; real task-averagedpass^k cannot generally be computed by exponentiating overallmean.", "producer_research_date": "2026-10-29" }, { "id": "E-060", "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "claim_type": "empirical_result", "location": "L33", "claim": "GPT4oaggregateunder50% andretailpass8under25%", "source_url": "https://arxiv.org/abs/2406.12045", "source_title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "source_date": "ICLR 2025", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_scope_clarification", "supporting_evidence": "Originalpaper fulltable2/consistencysection confirms aggregate48.2,retail61.2single,retailall8<25. Draft separates retail reliability clause but should make aggregate scope explicit.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-061", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "empirical_result", "location": "L17", "claim": "HELM17.9coverage andsevenmetrics", "source_url": "https://arxiv.org/abs/2211.09110", "source_title": "Holistic Evaluation of Language Models", "source_date": "TMLR 2023; arXiv revised 1 October 2023", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Originalabstract confirms design and historical coverage.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-062", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "empirical_result", "location": "L26", "claim": "GPQA448questions,65/74expert,34nonexpert,diamond198", "source_url": "https://arxiv.org/abs/2311.12022", "source_title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "source_date": "2023-11-20", "accessed_at_utc": "2026-10-29", "verdict": "supported_with_scope_clarification", "supporting_evidence": "Abstract supports headline65/74/34; fullsection2.3 table2 confirms448/198 but its main-subset validators are71.9/30.4 and diamond81.3/22.1, with selection caveats. Clarify headline figures are not the selected main-set table.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-063", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "empirical_result", "location": "L30", "claim": "MMLU-Pro10options,16–33pointdrops,24promptstylesandlowerpromptvariation", "source_url": "https://arxiv.org/abs/2406.01574", "source_title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "source_date": "NeurIPS 2024; arXiv v6, 6 November 2024", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Originalabstract and fullsection6.3 support those scoped results.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-064", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "vendor_reported_benchmark_result", "location": "L32", "claim": "Anthropicpostcontainstwo30/40SWEbaselinephrases", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "PrimarypostL182and327containboth; draft accurately notes discrepancy and retains only qualitative saturation.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-065", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "changing_benchmark_audit_result", "location": "L34", "claim": "February27.6/59.4auditsubset;July200/249andProretraction", "source_url": "https://openai.com/index/separating-signal-from-noise-in-coding-evaluations/", "source_title": "Separating signal from noise in coding evaluations", "source_date": "2026-07-08", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Both originalauditpages checked:Februaryfailure-selectedsample andJulyL40/L74 confirm bounded claims.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "E-066", "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "claim_type": "changing_benchmark_specification", "location": "L38", "claim": "BashOnlymini-SWE-agentversusfullagentview", "source_url": "https://www.swebench.com/verified.html", "source_title": "SWE-bench Verified", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "supported", "supporting_evidence": "Live maintainerL40–43statesdistinction; quoted no-tools sentence is the maintainer’s wording, not reviewer certification that bash invokes no tools.", "limits": "Claim-relevant primary passages checked; no study or provider service was reproduced. Retrieval day has UTC day precision; source date is distinct.", "producer_research_date": "2026-10-29" }, { "id": "B-035", "cluster": "B", "article_file": "articles/09-latency-cost.md", "claim_type": "methodological", "location": "L19–21,55", "claim": "End-to-end latency includes components; streaming onset and completion answer different questions", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_title": "Latency optimization", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official guide’s request/generation/streaming framework supports the distinction; no measured timing is asserted.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-28" }, { "id": "C-036", "cluster": "C", "article_file": "articles/latency-cost.md", "claim_type": "methodological", "location": "L37,41", "claim": "First useful streaming output and task completion must be distinguished", "source_url": "https://developers.openai.com/api/docs/guides/latency-optimization", "source_title": "Latency optimization", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official streaming/perceived-latency guidance supports separate clocks; the draft correctly treats required completeness as a quality condition.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-036", "cluster": "D", "article_file": "articles/ai-model-benchmarking-guide.md", "claim_type": "methodological", "location": "L61", "claim": "Final holdout loses untouched-test status when failures drive tuning", "source_url": "https://scikit-learn.org/stable/modules/cross_validation.html", "source_title": "Cross-validation: evaluating estimator performance", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official introduction explains test-set tuning leakage and separate validation/final test; draft retires the exposed evidence rather than certifying a failure-selected remainder.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-037", "cluster": "D", "article_file": "articles/llm-evaluation-metrics-rubrics.md", "claim_type": "methodological", "location": "L21–23", "claim": "Format success cannot establish semantic correctness or argument identity", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official agent section separates tool selection from argument precision; logical example of well-formed but wrong eligibility preserves task-specific scoring.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-038", "cluster": "D", "article_file": "articles/human-reviewers-llm-judges.md", "claim_type": "methodological", "location": "L73–75", "claim": "Allow insufficient-evidence verdicts and audit ordinary passes as well as flagged failures", "source_url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "source_title": "Demystifying evals for AI agents", "source_date": "2026-01-09", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Primary post L313 supplies abstention/Unknown guard; reviewing only selected negatives cannot expose unflagged false passes. Audit extension is explicitly methodological inference.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-039", "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "claim_type": "methodological", "location": "L23,63", "claim": "Repeats/related task variants cannot be counted as independent new conversations", "source_url": "https://arxiv.org/html/2411.00640v1", "source_title": "Adding Error Bars to Evals", "source_date": "2024-11-01", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Full paper clustered-standard-error and repeated-sampling sections support distinguishing question sampling from within-question variance.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "D-040", "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "claim_type": "methodological", "location": "L73", "claim": "Selecting maximum held-out score after many prompt trials risks selection luck", "source_url": "https://scikit-learn.org/stable/modules/cross_validation.html", "source_title": "Cross-validation: evaluating estimator performance", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "consistent_methodological_inference", "supporting_evidence": "Official test-set-overfitting explanation supports protected final evaluation after development selection; no effect size is claimed.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-22" }, { "id": "E-067", "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "claim_type": "methodological", "location": "L53", "claim": "A recent private email is certified clean relative to a published knowledge cutoff", "source_url": "https://developers.openai.com/api/docs/guides/evaluation-best-practices", "source_title": "Evaluation best practices", "source_date": "undated live documentation", "accessed_at_utc": "2026-10-29", "verdict": "overstated_guidance", "supporting_evidence": "The cutoff is useful provenance but does not prove absence from all subsequent training/fine-tuning or provider retention. Private status and exposure history need separate evidence; the draft overstates certification.", "limits": "Claim-directed live primary-source passages checked in source-retrieval evidence. Reviewer inference is distinguished from a source empirical finding; no benchmark or service experiment was run.", "producer_research_date": "2026-10-29" } ], "bibliographic_checks": [ { "cluster": "C", "article_file": "articles/human-evaluation-model-judges.md", "claim_type": "bibliographic_only", "location": "L35", "claim": "Zheng et al., 2024 labels arXiv:2406.07791", "source_url": "https://arxiv.org/abs/2406.07791", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_date": "2024-06-12; revised 2025-11-11", "accessed_at_utc": "2026-10-29", "verdict": "wrong_author_metadata", "supporting_evidence": "Live arXiv record lists Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma and Soroush Vosoughi. The linked paper exists and supports position-bias substance.", "limits": "Does not count toward substantive claim totals; no inference of fabrication or producer identity." }, { "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "claim_type": "bibliographic_only", "location": "L39", "claim": "Zheng et al., 2024 labels arXiv:2406.07791", "source_url": "https://arxiv.org/abs/2406.07791", "source_title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "source_date": "2024-06-12; revised 2025-11-11", "accessed_at_utc": "2026-10-29", "verdict": "wrong_author_metadata", "supporting_evidence": "Live arXiv record lists Lin Shi, Chiyu Ma, Wenhua Liang, Xingjian Diao, Weicheng Ma and Soroush Vosoughi. The linked paper exists and supports position-bias substance.", "limits": "Does not count toward substantive claim totals; no inference of fabrication or producer identity." } ], "counting_policy": "Claims are material factual/methodological review units, not independent experiments. Bibliographic checks are separate. Hypothetical arithmetic is explicitly tagged and separately counted; every article also has at least3 non-hypothetical checks. Composite pricing rows group closely related rate-card assertions, all checked against relevant primary tables." }