{ "status": "final_independent_stage1", "articles": [ { "cluster": "A", "article_file": "articles/evaluating-ai-models-and-agent-systems.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L22–36 accurately ground the overview in MMLU, HELM and historical agent studies. L90 labels its arithmetic hypothetical. L34 overstates the official workflow taxonomy: the guide says workflows add no new elements of nondeterminism. Primary-source checks: fact-check-ledger.json A-001, A-002, A-003, A-004, A-005, A-006, A-007, A-008, A-009, A-010, A-011, A-056.", "practical_usefulness": "L40–56 gives a usable six-step decision loop; L78–98 supplies trial/sample and decision-memo details. Fixed sample/trial suggestions need clearer workload qualification, and L106 overpromises how many expensive mistakes 100 tasks can avoid.", "writing_quality": "L14–18 sets a concrete problem and scope; the sequence remains readable despite dense benchmark lists at L22–36. Repeated absolute slogans at L28,52,100,106 need a light precision edit.", "intent_and_metadata": "L2–9 provides a specific production-evaluation query, descriptive title, role and accurate description; cluster-map ai-eval-00 matches the article’s purpose. No invented search-demand figures." }, "publication_blockers": [], "fact_check_ids": [ "A-001", "A-002", "A-003", "A-004", "A-005", "A-006", "A-007", "A-008", "A-009", "A-010", "A-011", "A-056" ] }, { "cluster": "A", "article_file": "articles/public-benchmarks-vs-workload-evaluation.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L22–42 uses relevant original papers with correctly bounded historical counts and Arena estimates. Some absolute statements at L48 and L58 overstate how little public evidence can transfer. Primary-source checks: fact-check-ledger.json A-012, A-013, A-014, A-015, A-016.", "practical_usefulness": "L62–84 gives a two-stage selection method and explicit shortlist columns; L92–104 provides vendor questions. L88–90 labels the failure case hypothetical. Add a completed scorecard row to make the template easier to apply.", "writing_quality": "L18–42 builds from construct to leaderboard process; L44–108 keeps a clear reader journey. Tighten repeated screening-versus-selection conclusions and categorical phrasing.", "intent_and_metadata": "L4–7 names the public-versus-custom question and promises the content delivered in L62–84. The proposed slug and description agree with cluster-map ai-eval-01." }, "publication_blockers": [], "fact_check_ids": [ "A-012", "A-013", "A-014", "A-015", "A-016" ] }, { "cluster": "A", "article_file": "articles/designing-evaluation-tasks-and-datasets.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L26,102,114,156 cite primary guidance and methodological papers; L106 recognises effect size and correlation. L110–112 sample-size bands are editorial heuristics and need stronger caveats against guaranteed decision-grade inference. Primary-source checks: fact-check-ledger.json A-017, A-018, A-019.", "practical_usefulness": "L49–75 task schema, L89–100 labelling protocol and L118–124 split/version policy are concrete. The claim at L14 that convenience/same-model synthetic cases produce no evidence is too absolute; sampling limitations need calibrated wording.", "writing_quality": "The progression from decision to provenance, sampling, oracle and labelling is easy to follow. L140–164 repeats earlier warnings and could be tightened without losing the checklist.", "intent_and_metadata": "L4–7 targets evaluation dataset design precisely; description and schema meet that intent. ID/path and planned unique purpose agree with cluster-map ai-eval-02." }, "publication_blockers": [], "fact_check_ids": [ "A-017", "A-018", "A-019" ] }, { "cluster": "A", "article_file": "articles/contamination-leakage-and-holdouts.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L36–40 faithfully reports real contamination studies. L44 conflates LiveCodeBench’s initial release range with its revised abstract. L88 incorrectly turns missing decontamination disclosure into an upper-bound claim. Primary-source checks: fact-check-ledger.json A-020, A-021, A-022, A-023, A-024.", "practical_usefulness": "L20–30 distinguishes four leakage mechanisms; L62–78 and L98–107 offer workable custody controls. L82–90’s diagnosis and upper-bound advice need substantive repair: distribution shift and unknown data do not establish the claimed bound.", "writing_quality": "L20–60 has a coherent threat-model structure and clear controls. Qualify sweeping language at L58 and L94 and avoid suggesting perturbation results uniquely identify memorisation.", "intent_and_metadata": "L4–7 addresses the precise contamination/leakage query; description honestly promises risk/control explanation and cluster-map ai-eval-03 matches." }, "publication_blockers": [ "L88: replace the unsupported deployment-performance upper-bound guidance; source disclosure uncertainty cannot establish that bound." ], "fact_check_ids": [ "A-020", "A-021", "A-022", "A-023", "A-024" ] }, { "cluster": "A", "article_file": "articles/objective-metrics-and-rubrics.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L30 gives the correct HumanEval estimator; L32 distinguishes all-success reliability and L38 separates tool choice/arguments. L74’s micro-average-as-traffic wording needs qualification when the sample oversamples risk slices. Primary-source checks: fact-check-ledger.json A-025, A-026, A-027.", "practical_usefulness": "L58–68 supplies a concrete gated support rubric and L78–109 a validation/metric checklist. Add estimator conditions n≥k and a numeric classification example; token F1 does not invariably reward verbosity as L44 implies.", "writing_quality": "L18–46 explains choices in compact terms and the worked rubric is legible. Some formula terms and strong slogans need a light explanatory edit.", "intent_and_metadata": "L4–7 accurately identifies metrics/rubrics intent; description names the families and validation method actually provided. Planned metadata agrees with ai-eval-04." }, "publication_blockers": [], "fact_check_ids": [ "A-025", "A-026", "A-027" ] }, { "cluster": "A", "article_file": "articles/human-evaluation-and-model-judges.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L38 and L72 accurately attribute criteria drift and study-specific >80% judge agreement/biases. L82 and L90 should not imply that order averaging or requiring reasoning guarantees sound judgments; the original paper presents conditional mitigations. Primary-source checks: fact-check-ledger.json A-028, A-029, A-030.", "practical_usefulness": "L42–66 covers rubric calibration and agreement; L98–110 offers a complete held-out judge-validation sequence. Include a small confusion matrix and false-pass audit rather than relying primarily on aggregate agreement.", "writing_quality": "L14–40 clearly frames humans and models as fallible instruments. The mitigation list is easy to scan; a few categorical claims (L28,82,86) need qualification.", "intent_and_metadata": "L4–7 identifies judge reliability, with a descriptive title and faithful description; cluster-map ai-eval-05 supplies the specific reader question and matching fields." }, "publication_blockers": [], "fact_check_ids": [ "A-028", "A-029", "A-030" ] }, { "cluster": "A", "article_file": "articles/repeatability-variance-and-uncertainty.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L32–58,66,82,88 accurately reference the checked formulas, examples and study results. L50–52 arithmetic is consistent. Normal intervals need clearer boundary/small-n qualifications, and the L96–100 table should be explicitly captioned hypothetical. Primary-source checks: fact-check-ledger.json A-031, A-032, A-033, A-034, A-035, A-036, A-037. Additional check A-058: L76 needs a domain qualifier: original tau-bench retail single-attempt GPT-4o was 61.2%; under 50% is the cross-domain average, whereas pass^8 below 25% is retail. Genuine source, minor scope correction.", "practical_usefulness": "Paired SE, clustering, trial variance and a power formula at L44–90 enable concrete planning. A worked task-level difference vector or implementation would make the grouped/repeated-run analysis easier to execute; no independent experimental result is claimed.", "writing_quality": "L18–90 progresses from sources of variation to calculations with useful signposts. The final failure lists repeat earlier cautions; explain limits of the normal approximation near zero/one.", "intent_and_metadata": "L4–7 matches statistical-significance/error-bar intent and a focused description; cluster-map ai-eval-06 promises this minimum toolkit and the draft supplies it." }, "publication_blockers": [], "fact_check_ids": [ "A-031", "A-032", "A-033", "A-034", "A-035", "A-036", "A-037", "A-058" ] }, { "cluster": "A", "article_file": "articles/prompt-tool-and-configuration-fairness.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L74 and L100 are supported by Inspect and SWE-bench maintainers. L110’s “always” and L114’s cross-model nominal-effort equivalence are not established by model-dependent adaptive effort documentation. Primary-source checks: fact-check-ledger.json A-038, A-039, A-040.", "practical_usefulness": "L37–84 gives a useful manifest and two legitimate prompt designs; L119–146 handles retries/failures. Correct L110–117 before using its matched-effort recipe: nominal settings do not create a common resource budget.", "writing_quality": "The manifest, controlled/adapted prompt distinction and audit are clearly arranged. L110–117 needs a more precise argument; much of the L131–146 checklist restates earlier sections.", "intent_and_metadata": "L4–7 and cluster-map ai-eval-07 align with fairness intent and complete metadata. “fair comparison of AI models prompts tools” needs a small natural-language query edit." }, "publication_blockers": [ "L110–117: revise the materially misleading matched-effort guidance and universal token/latency claim; nominal settings are model-dependent and do not establish equal resources." ], "fact_check_ids": [ "A-038", "A-039", "A-040" ] }, { "cluster": "A", "article_file": "articles/latency-token-usage-and-cost.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L68 dates the price snapshot; current official rows corroborate the stated Astra/Luna, Fable and Gemini figures. L75 omits Haiku’s ≤100K input tier. L84–104 arithmetic is explicitly hypothetical and checks out. Primary-source checks: fact-check-ledger.json A-041, A-042, A-043, A-044, A-045, A-046, A-047, A-057.", "practical_usefulness": "L24–55 defines latency/accounting and L84–116 works through cost and a quality frontier. State that output in the cost formula includes billed reasoning, and measure cold as well as warmed-cache performance at L135.", "writing_quality": "The worked arithmetic is transparent and tables are usable. L118–129 duplicates L57–64 hidden-cost items and can be consolidated; tier details belong beside the prices.", "intent_and_metadata": "L4–7 clearly targets task economics/latency, and metadata agrees with ai-eval-08. The “other half” title is slightly less direct than the otherwise specific query and description." }, "publication_blockers": [], "fact_check_ids": [ "A-041", "A-042", "A-043", "A-044", "A-045", "A-046", "A-047", "A-057" ] }, { "cluster": "A", "article_file": "articles/evaluating-agents-and-tool-use.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L37–43 reports genuine benchmark methods and historical results; current subset specifications agree with the maintainer. L73 correctly attributes the trace-to-dataset workflow and L79 marks numbers hypothetical. Primary-source checks: fact-check-ledger.json A-048, A-049, A-050, A-051, A-052.", "practical_usefulness": "L18–29 decomposition, L47–73 outcome/trajectory measures, L81–106 reset/stub/ablation protocol and L108–129 failure/safety reporting form a particularly complete practical method. Budget and policy violations are explicit rather than hidden inside success.", "writing_quality": "The distinction between outcome and trajectory is clear and the protocol is concrete. Tighten the repeated claim that any single success is only an anecdote and clarify finite-k reliability’s deployment interpretation.", "intent_and_metadata": "L4–7 and cluster-map ai-eval-09 focus exactly on agents/tool-use evaluation; title and description name the system distinction and concrete measures actually delivered." }, "publication_blockers": [], "fact_check_ids": [ "A-048", "A-049", "A-050", "A-051", "A-052" ] }, { "cluster": "A", "article_file": "articles/interpreting-results-and-maintaining-evals.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L45 and L75 accurately use BetterBench/Leaderboard Illusion; L112’s scheduled platform dates match current official notice. L41’s description of significance should be softened: a test does not establish existence/nonexistence of an effect. Primary-source checks: fact-check-ledger.json A-053, A-054, A-055.", "practical_usefulness": "L18–31 gives interpretation questions, L79–94 a decision memo and L98–114 lifecycle actions. Specify which holdout is sealed versus familiar when scheduling regression reruns; the cadence is editorial advice rather than a measured optimum.", "writing_quality": "The reader moves naturally from result to decision to maintenance. Repeated absolute contrasts and the L116–126 maturity model add some generic management prose; minor tightening is sufficient.", "intent_and_metadata": "L4–7 and ai-eval-10 match interpretation/maintenance intent with a descriptive title, specific description and useful operational reader question." }, "publication_blockers": [], "fact_check_ids": [ "A-053", "A-054", "A-055" ] }, { "cluster": "B", "article_file": "articles/01-benchmarking-models.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L23,29,33,35,51 cite relevant primary benchmark/method sources and bound the claims carefully. L17 explicitly labels the running example; configuration and sampling limitations are well distinguished. Primary-source checks: fact-check-ledger.json B-001, B-002, B-003, B-004.", "practical_usefulness": "L19–77 gives a complete decision journey and L81–85 a bounded first-week plan. It needs a filled case/run scorecard or numerical analysis example to reach excellent technical depth, but the workflow is usable.", "writing_quality": "The defined fictional assistant makes the argument concrete and the progression is clear. Some extended caution paragraphs and repeated reference to the policy assistant could be shortened.", "intent_and_metadata": "L2–8 has a strong broad query, descriptive decision-oriented title and accurate description; B/cluster-map.json record 01 matches the stated reader question." }, "publication_blockers": [], "fact_check_ids": [ "B-001", "B-002", "B-003", "B-004" ] }, { "cluster": "B", "article_file": "articles/02-public-benchmarks.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21,31,37 cite maintainer/original material supporting task/denominator, standardisation and exposure claims. The draft does not turn uncertainty into accusations or a claimed current ranking. Primary-source checks: fact-check-ledger.json B-005, B-006, B-007.", "practical_usefulness": "L17–47 teaches task/configuration/coverage inspection and L55–59 provides actionable vendor questions. A filled benchmark card with a real denominator would increase depth without needing a ranking.", "writing_quality": "Clear sections build from score meaning to an experiment and procurement questions. The final vendor section repeats several earlier cautions and can be tightened.", "intent_and_metadata": "L4–7 naturally targets reading AI benchmarks and the promised title/description is fulfilled; cluster-map 02 identifies a distinct transfer question." }, "publication_blockers": [], "fact_check_ids": [ "B-005", "B-006", "B-007" ] }, { "cluster": "B", "article_file": "articles/03-task-design.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L23 uses current primary evaluation guidance; L43,51,57 correctly qualify weighting, synthetic references and grouped cases. Many later methodological recommendations lack nearby supporting citations, but are presented as bounded design advice. Primary-source checks: fact-check-ledger.json B-008, B-009, B-010.", "practical_usefulness": "L25–37 supplies a specific missing-evidence case and critical failure; L39–65 addresses slices, provenance, grouping, pilot checks and exclusions. A schema or completed coverage table would make this stronger for technical implementers.", "writing_quality": "The workflow-to-case progression is readable and the contractor example is clear. The last lifecycle section is somewhat generic and repeats provenance/version advice.", "intent_and_metadata": "L2–8 and cluster-map 03 agree on task-design intent; title promises a traceable dataset and the example fulfils that promise without fabricated search figures." }, "publication_blockers": [], "fact_check_ids": [ "B-008", "B-009", "B-010" ] }, { "cluster": "B", "article_file": "articles/04-datasets-holdouts.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21’s exposure/reproducibility attribution is supported, and L25–37 carefully distinguishes grouping, time splits and observability. Grouped splitting and similarity limitations would benefit from nearby sources in addition to the one contamination paper. Primary-source checks: fact-check-ledger.json B-011, B-012, B-013.", "practical_usefulness": "L23–45 gives operational partition/access/reference rules; L47–57 handles refresh and exposure without quietly deleting unfavourable cases. Add a small concrete partition manifest and leak audit to deepen implementation.", "writing_quality": "Clear, measured prose separates team leakage from training contamination. Some provenance/access/version advice recurs across L31–45 and the ending could be tighter.", "intent_and_metadata": "L4–7 gives a natural holdout query, focused title and accurate description; cluster-map 04 preserves the distinct boundary/lifecycle purpose." }, "publication_blockers": [], "fact_check_ids": [ "B-011", "B-012", "B-013" ] }, { "cluster": "B", "article_file": "articles/05-metrics-rubrics.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21–29 correctly distinguishes denominators, precision/recall and syntax/semantic checks; official definitions and eval guidance support them. L41–49 labels gates/weights as choices rather than experimental facts. Primary-source checks: fact-check-ledger.json B-014, B-015, B-016.", "practical_usefulness": "L33–43 provides observable criteria and pass rules; L59–63 works through boundary judgments. A fully scored matrix with numeric metric calculation would add technical depth, but readers can apply the rubric now.", "writing_quality": "The distinction between metric, rubric and decision remains clear; examples at L61–63 concretely explain omissions and abstentions. The composite and validation sections could be slightly more concise.", "intent_and_metadata": "L4–7 matches metrics/rubrics intent and the description’s denominator promise; cluster-map 05 and article metadata agree." }, "publication_blockers": [], "fact_check_ids": [ "B-014", "B-015", "B-016" ] }, { "cluster": "B", "article_file": "articles/06-human-model-judges.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L31 accurately limits the position-bias study to its tested judges/tasks, and L33,37–39 correctly treats style and calibration risks. More primary citations should sit beside self-enhancement and grader-injection claims. Primary-source checks: fact-check-ledger.json B-017, B-018, B-019.", "practical_usefulness": "L17–47 specifies independent review, ties, evidence visibility, held-out judge validation and false-pass inspection; L51–55 includes audit capacity. It lacks an actual judge prompt or confusion matrix, limiting depth rather than usability.", "writing_quality": "Nuanced prose consistently separates agreement from truth and explanation from proof. The repeated calibration and independent-label instructions can be consolidated.", "intent_and_metadata": "L4–7 and cluster-map 06 deliver a distinct human/model-judge question with accurate descriptive metadata." }, "publication_blockers": [], "fact_check_ids": [ "B-017", "B-018", "B-019" ] }, { "cluster": "B", "article_file": "articles/07-repeatability-uncertainty.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L29’s paired/group resampling logic and L31’s small-sample binomial caution are supported by primary software/statistical documentation. L35 correctly avoids equating nonsignificance with equivalence. Bootstrap details lack a direct in-article reference. Primary-source checks: fact-check-ledger.json B-020, B-021, B-022.", "practical_usefulness": "L25–39 describes pairing and independent units clearly, but the invented 82/78 example never supplies paired outcomes, computes an interval or shows a complete bootstrap procedure. Ordinary substantive editing should add an executable or fully numeric worked example.", "writing_quality": "The distinction among case, run and environmental variation is clear. The article remains largely advisory, with repeated statements about insufficient evidence instead of developing the example.", "intent_and_metadata": "L4–7 focuses on repeatability/confidence intervals; title and description accurately reflect the cautious paired-analysis discussion, and cluster-map 07 matches." }, "publication_blockers": [], "fact_check_ids": [ "B-020", "B-021", "B-022" ] }, { "cluster": "B", "article_file": "articles/08-fair-comparisons.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L19–29 correctly separates controlled substitution from tailored systems and records compatibility gaps; L35,39 correctly counts resources and missing outputs. HELM support is relevant but most budget/control guidance is uncited editorial method. Primary-source checks: fact-check-ledger.json B-023, B-024, B-025.", "practical_usefulness": "L25–45 supplies concrete manifest fields, resource rules, execution order and tuning boundaries. L55–59 handles native-interface/context gaps usefully; a filled manifest would reduce reader assembly work.", "writing_quality": "Clear structure makes the two experiment questions easy to distinguish. Resource and tuning cautions repeat in adjacent sections and the last paragraph is dense.", "intent_and_metadata": "L4–7 naturally targets fair comparisons when prompts/tools differ, exactly the draft’s distinctive question; cluster-map 08 agrees with metadata." }, "publication_blockers": [], "fact_check_ids": [ "B-023", "B-024", "B-025" ] }, { "cluster": "B", "article_file": "articles/09-latency-cost.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L15 clearly states invented rates and L29–31 arithmetic is correct. However the article has no external citation for billable reasoning/categories, latency tails or load guidance; its brief points to sources in an overview that itself is not directly linked here. Add primary accounting/measurement support. Primary-source checks: fact-check-ledger.json B-026, B-027, B-028.", "practical_usefulness": "L17–31 defines useful clocks and works cost per attempt/success; L33–57 discusses workload, load, tails and sensitivity. The calculation is followable, though a measurement record and numeric tail example would improve depth.", "writing_quality": "Concrete arithmetic and a useful slow-tail diagnosis keep the argument readable. Several paragraphs repeat warnings about keeping quality stable; light tightening would help.", "intent_and_metadata": "L4–7 accurately targets latency/token/cost evaluation and promises the practical user-visible boundary delivered; cluster-map 09 has a distinct economics purpose." }, "publication_blockers": [], "fact_check_ids": [ "B-026", "B-027", "B-028", "B-035" ] }, { "cluster": "B", "article_file": "articles/10-agent-evaluation.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L15 and L31 use appropriate primary benchmark/maintainer sources; L19 labels the expense setting fictional. L29–45 distinguishes logs, actual state and prohibited actions without claiming universal sandbox safety. Primary-source checks: fact-check-ledger.json B-029, B-030, B-031.", "practical_usefulness": "L17–49 specifies start/goal/authority, trace fields, outcome/process gates and component tests. L55–59’s temporary-read/partial-write failures are concrete, but a fully specified state assertion or trace example would strengthen the method.", "writing_quality": "The final-message versus action distinction is clear and the injected-failure examples add specificity. Trim repeated sandbox-authorisation cautions and define a trace schema for easier scanning.", "intent_and_metadata": "L4–7 and cluster-map 10 focus precisely on agent tools, outcome and authority; title/description match the actual failure-injection coverage." }, "publication_blockers": [], "fact_check_ids": [ "B-029", "B-030", "B-031" ] }, { "cluster": "B", "article_file": "articles/11-results-maintenance.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L41’s continuous evaluation/calibration attribution is supported. L39 and L45–49 appropriately retain dependence and changed-instrument limitations; most maintenance actions are clearly recommendations rather than research results. Primary-source checks: fact-check-ledger.json B-032, B-033, B-034.", "practical_usefulness": "L17–33’s layered scorecard and decision options are usable; L35–61 supplies incident/version/review triggers and unavailable-baseline handling. A completed decision record would add depth and reduce repetition.", "writing_quality": "The sequence from run validity through gates, ordinary quality and lifecycle is coherent. Several later paragraphs restate preservation/versioning rather than develop one complete change review.", "intent_and_metadata": "L4–6 is accurate and informative, but target_query “interpret maintain AI evaluations” at L7 is awkward and cluster-map uses a broad generic search-intent label. Minor metadata editing is needed." }, "publication_blockers": [], "fact_check_ids": [ "B-032", "B-033", "B-034" ] }, { "cluster": "C", "article_file": "articles/agent-system-evaluation.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L33 correctly attributes trace grading and L43 accurately describes the scoped 2026 SWE-Bench Pro Verified preprint; no benchmark result is invented. L55–57 explicitly labels the repository example hypothetical. Checked primary-source evidence: fact-check-ledger.json C-001, C-002, C-003.", "practical_usefulness": "L17–41 defines boundaries, external outcome tests and environment controls; L45–67 covers recovery, cost, ablations and task-bank history. Add a concrete trace or state assertion for implementation; the existing example is usable but remains prose.", "writing_quality": "The outcome/trajectory distinction is clear, especially L35’s separation of observable record from explanation. L61–71 repeats earlier boundary and fairness points and could be tightened.", "intent_and_metadata": "L4–8 gives focused agent/tool intent and an accurate description; cluster-map agrees. Proposed slug at L5 includes uppercase AI and should be normalised for a consistent URL convention." }, "publication_blockers": [], "fact_check_ids": [ "C-001", "C-002", "C-003" ] }, { "cluster": "C", "article_file": "articles/build-task-eval.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "The labelled support-ticket example and dependence/sampling caveats are sound, but the article contains no external citation despite substantive claims about grouped splitting, sample size and reviewer agreement. Its evidence-plan citations appear only in supporting records. Checked primary-source evidence: fact-check-ledger.json C-004, C-005, C-006.", "practical_usefulness": "L17–43 defines a concrete task contract and labelling workflow; L69–73 shows 120 cases/30 holdout with an explicit illustrative caveat. A completed case row and benchmark matrix would strengthen technical execution without changing the method.", "writing_quality": "The ticket-routing spine is coherent and the checklist is concrete. Several later audit/privacy paragraphs repeat generic versioning principles; a light structural edit would improve focus.", "intent_and_metadata": "L4–8 promises a task-centred design method and supplies it. Slug/query/description and cluster-map task-evaluation reader question align, without invented search data." }, "publication_blockers": [], "fact_check_ids": [ "C-004", "C-005", "C-006" ] }, { "cluster": "C", "article_file": "articles/choosing-model-benchmark.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L33 accurately bounds MMLU/HELM, L76 uses official pricing structure without asserting rates, and L84 correctly describes issue repair. L15 and L88 clearly label invented thresholds. More direct method citations would improve traceability for the uncertainty section. Checked primary-source evidence: fact-check-ledger.json C-007, C-008, C-009.", "practical_usefulness": "L17–90 covers decisions, cases, scorer, comparison type, operations and maintenance with an invoice contract and explicit release rule. A completed scorecard or computed interval would improve depth; the current framework is actionable.", "writing_quality": "The invoice example and defined gates make the long guide easy to follow. Cost, boundary and versioning cautions recur in several sections, and the conclusion repeats the opening premise.", "intent_and_metadata": "L4–8 and cluster-map model-benchmarking clearly target workload benchmarking; a specific title, accurate description and broad entry query fulfil the pillar role." }, "publication_blockers": [], "fact_check_ids": [ "C-007", "C-008", "C-009" ] }, { "cluster": "C", "article_file": "articles/datasets-holdouts-contamination.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L31’s 2026 preprint exists and the scope caveat is faithful. The nearby Yang citation characterises a black-box original/rewrite audit, whereas the paper’s core method trains on rephrased tests and detects overlap in corpora. That attribution needs substantive correction. Checked primary-source evidence: fact-check-ledger.json C-010, C-011, C-012.", "practical_usefulness": "L17–49 distinguishes generalisation targets, storage boundaries, partitions and version bridges; L53–66 offers a concrete contamination review. L60’s runner/label boundary and L68–72 freshness limitations are particularly useful; add a small partition manifest.", "writing_quality": "The sections follow a clear data-integrity lifecycle and avoid asserting zero contamination. The long L31 citation paragraph should be separated and clarified to keep two different study methods distinct.", "intent_and_metadata": "L4–8 and cluster-map dataset-integrity match the leakage query with accurate split/provenance/version promises. The article’s local links give clear next steps." }, "publication_blockers": [], "fact_check_ids": [ "C-010", "C-011", "C-012" ] }, { "cluster": "C", "article_file": "articles/fair-model-comparisons.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L19–41 gives appropriately qualified controlled-versus-configured experiments and API asymmetry. However, there are no external citations for prompt effects, tool/budget confounding or uncertainty guidance; the brief’s evidence plan is not a near-text source. Checked primary-source evidence: fact-check-ledger.json C-013, C-014, C-015.", "practical_usefulness": "L23–45 supplies concrete controls and L49–59 shows two incident-summary designs plus ledger fields. A filled ledger would improve implementation speed; the distinction between product selection and attribution is useful as written.", "writing_quality": "The two-design distinction is consistently expressed and the incident-summary example is labelled hypothetical. Repeated freeze/log/disclose instructions across adjacent sections could be consolidated.", "intent_and_metadata": "L4–8 and cluster-map fair-comparison have a natural query, focused title, faithful description and a distinct reader question. No unverified keyword-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "C-013", "C-014", "C-015" ] }, { "cluster": "C", "article_file": "articles/human-evaluation-model-judges.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L35 links to a real position-bias paper that supports the substance, but wrongly calls it Zheng et al.; it is Shi and colleagues. L51 correctly attributes criteria/examples and calibration to official guidance; examples are explicitly illustrative. Checked primary-source evidence: fact-check-ledger.json C-016, C-017, C-018.", "practical_usefulness": "L23–55 offers anchors, independent calibration, bias controls, false-pass review and agreement reporting. L73–75 defines a useful hybrid deployment pattern. Add a full graded example and separate calibration/validation sets explicitly.", "writing_quality": "Clear discussion separates agreement, truth and adjudication; L55 correctly asks for method/prevalence context. The last three sections repeat calibration/versioning points and could be tightened.", "intent_and_metadata": "L4–8 and cluster-map human-judging accurately address human/model-judge choice with focused metadata and no fabricated demand evidence." }, "publication_blockers": [], "fact_check_ids": [ "C-016", "C-017", "C-018" ] }, { "cluster": "C", "article_file": "articles/latency-cost.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L29’s official pricing-category claim is supported and avoids stale rate tables. L57’s $258/$127 hypothetical arithmetic is correct. Clarify L27 so retry charges cannot be double-counted within already-total input/output charges and add near-text latency support. Checked primary-source evidence: fact-check-ledger.json C-019, C-020, C-021.", "practical_usefulness": "L17–53 defines success, accounting/timing boundaries, load and fallback treatment. L57’s reviewer-cost example is specific and followable, though it stops before explicitly dividing by successful/corrected task counts.", "writing_quality": "The prose is clear and the human-review calculation gives the article a concrete centre. Several cost/assumption paragraphs restate each other; make the displayed formula’s units and exclusions more precise.", "intent_and_metadata": "L4–8’s title/description are faithful and the query is useful; cluster-map latency-cost matches. Proposed slug begins uppercase AI and needs normalisation." }, "publication_blockers": [], "fact_check_ids": [ "C-019", "C-020", "C-021", "C-036" ] }, { "cluster": "C", "article_file": "articles/maintaining-evaluations.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L25 accurately cites official continuous-evaluation guidance. L38–46 correctly separates changed suites, task mix and grader versions. Counts at L56 are labelled hypothetical rather than validated sample prescriptions. Checked primary-source evidence: fact-check-ledger.json C-022, C-023, C-024.", "practical_usefulness": "L17–46 offers useful release/version, smoke/regression/holdout and bridge roles; L48–68 assigns decisions and owners. The routine is usable but would benefit from a completed changelog/bridge report rather than another checklist.", "writing_quality": "The lifecycle is easy to follow and L34 explains dataset roles clearly. The ending checklist and conclusion repeat earlier versioning instructions; tighten those passages.", "intent_and_metadata": "L4–8 promises maintenance/version comparability and delivers it; cluster-map agrees. Uppercase LLM in proposed_slug is an inconsistent URL style requiring a minor edit." }, "publication_blockers": [], "fact_check_ids": [ "C-022", "C-023", "C-024" ] }, { "cluster": "C", "article_file": "articles/metrics-rubrics.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21’s precision/recall definitions and L31/L45’s official eval/grader references are supported. L35’s illustrative groundedness anchors overlap: a minor unsupported detail also meets the level-2 definition that all material claims are supported. Checked primary-source evidence: fact-check-ledger.json C-025, C-026, C-027, C-028.", "practical_usefulness": "L17–29 and L49–75 offer useful field/record, gates, selective-coverage and weighted-score guidance. The main rubric example at L35 needs substantive repair into exclusive levels, plus a scored output example before readers can apply it consistently.", "writing_quality": "The article clearly distinguishes metrics, scorecards and rubrics, with sensible terminology. The ambiguous central anchor wording needs a precision edit; later checklist prose repeats existing cautions.", "intent_and_metadata": "L4–8 matches metrics/rubrics intent and the title/description accurately promise task outcomes, costs and gates. Cluster-map evaluation-metrics agrees with the metadata." }, "publication_blockers": [], "fact_check_ids": [ "C-025", "C-026", "C-027", "C-028" ] }, { "cluster": "C", "article_file": "articles/public-benchmark-literacy.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L21/L31 accurately describe MMLU/HELM and L39’s position-bias substance is supported, but the author label is wrong. L45’s Yang method is imprecisely recast as a direct memorisation probe; correct both attributions while retaining the bounded contamination warning. Checked primary-source evidence: fact-check-ledger.json C-029, C-030, C-031, C-032.", "practical_usefulness": "L17–51 examines task, comparability, aggregation, judging and system boundaries; L53–72 supplies a shortlist table structure and checklist. Add a filled real benchmark card to demonstrate how to use the otherwise sound method.", "writing_quality": "The sections are clear and carefully avoid universal intelligence claims. Several screening/selection cautions repeat the pillar and the last paragraph can be shorter.", "intent_and_metadata": "L4–8 is faithful to public-score interpretation with a useful query; cluster-map matches. Uppercase AI in the proposed slug needs the same URL-style normalisation as other C records." }, "publication_blockers": [], "fact_check_ids": [ "C-029", "C-030", "C-031", "C-032" ] }, { "cluster": "C", "article_file": "articles/repeatability-uncertainty.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L30 correctly retains case dependence under repeats, and L42’s resampling-unit advice is sound. Bouthillier is a real training-pipeline variance study; a closer grouped/bootstrap source would better support the precise hosted-evaluation procedure. Checked primary-source evidence: fact-check-ledger.json C-033, C-034, C-035.", "practical_usefulness": "L23–46 explains repeat/case distinctions and paired/grouped analysis; L60–73 provides a lightweight protocol. L77’s 82/85 example never supplies discordant counts or computes an interval, leaving the key analysis step for readers to invent.", "writing_quality": "Clear, measured wording avoids overclaiming reproducibility or overlapping intervals. Much of the article remains advisory; developing the numeric example would replace repeated general cautions.", "intent_and_metadata": "L4–8 and cluster-map repeatability accurately describe confidence intervals/repeats. Uppercase AI at the start of proposed_slug needs normalisation before publication." }, "publication_blockers": [], "fact_check_ids": [ "C-033", "C-034", "C-035" ] }, { "cluster": "D", "article_file": "articles/ai-model-benchmarking-guide.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L33 accurately describes MMLU; L73 correctly attributes task-specific, human-calibrated evaluation guidance. L105 labels the $12/80 illustration and computes it correctly. Some broader process recommendations could carry closer source links. Primary checks: D-005, D-006, D-007.", "practical_usefulness": "The support-assistant decision contract and six-artefact close (L123) make the overall method usable; the guide connects gates, sampling, scorer, operations and maintenance. A filled decision report would make the long framework easier to implement.", "writing_quality": "Clear distinctions and a stable example guide the reader. Several control/version/report checklists repeat; tighten them and replace a few abstract instructions with a completed artefact.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-005", "D-006", "D-007", "D-036" ] }, { "cluster": "D", "article_file": "articles/read-ai-benchmark-leaderboards.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "HumanEval at L23/35, the dated SWE-bench scaffold explanation at L43 and HELM at L53 faithfully match the primary sources. The draft carefully scopes any-of-k and avoids current rank claims. Primary checks: D-030, D-031, D-032.", "practical_usefulness": "The evidence-card fields at L79 and the task/harness/date/metric screening procedure are actionable. Add one filled card so the reader sees how a real ambiguous leaderboard row becomes a shortlist decision.", "writing_quality": "Readable progression from benchmark task to score boundary to evidence card. Several cautions repeat the pillar, and the closing checklist could be shorter.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-030", "D-031", "D-032" ] }, { "cluster": "D", "article_file": "articles/design-llm-evaluation-tasks.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "CheckList at L55 is accurately characterised. L57 and L65–75 correctly distinguish challenge sampling and clustered variants from independent traffic; these are bounded editorial methods rather than experimental findings. Primary checks: D-011, D-012, D-013.", "practical_usefulness": "RETURN-014 gives a specific missing-delivery-date failure, case/provenance fields at L41–47, a two-axis coverage plan and contrastive/invariance tests at L65–67. A completed label/example output would improve the otherwise usable card.", "writing_quality": "The single running case connects the method well. The pilot and sample-variation ending adds relevant cautions but repeats some earlier freeze-and-record advice.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-011", "D-012", "D-013" ] }, { "cluster": "D", "article_file": "articles/evaluation-dataset-leakage-holdouts.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21 faithfully uses the official leakage definition; L23 scopes pretraining detection; L31 accurately adapts Datasheets; L65 correctly states the canonical/shuffled test and its exchangeability assumption. No test is presented as a universal contamination certificate. Primary checks: D-014, D-015, D-016, D-017.", "practical_usefulness": "L27–61 gives provenance, grouped partitions, exposure records and overlap review with sensible limitations. Add a filled manifest/overlap decision to make the procedure immediately executable.", "writing_quality": "Precise distinctions among leakage, exposure and evidence avoid overclaiming. Some record/version advice repeats adjacent articles and can be tightened.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-014", "D-015", "D-016", "D-017" ] }, { "cluster": "D", "article_file": "articles/llm-evaluation-metrics-rubrics.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L39–43 confusion-matrix arithmetic is correct: precision .60, recall .75, F1 about .667, accuracy .85. L47 correctly distinguishes macro and weighted views; L51–67 bounds the illustrative rubric. Primary checks: D-024, D-025, D-026.", "practical_usefulness": "The completed TP/FP/FN/TN example, baseline comparison and separate non-negotiable gates demonstrate how to choose and interpret the metric. The anchors and aggregation tradeoffs are specific enough to apply.", "writing_quality": "The worked example clarifies the abstract scoring advice. The last checklist repeats a few points; add one boundary output during final publication checks for rubric clarity.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-024", "D-025", "D-026", "D-037" ] }, { "cluster": "D", "article_file": "articles/human-reviewers-llm-judges.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L49 accurately scopes MT-Bench biases; independent fresh validation at L33/43 is methodologically sound. The L61 always-pass counterexample correctly shows 90% agreement and zero failure detection. Primary checks: D-021, D-022, D-023.", "practical_usefulness": "Calibration, fresh validation, false-pass review and stratified ongoing audits are useful and specific. A completed adjudicated example or threshold choice would deepen implementation without changing the sound procedure.", "writing_quality": "Clear separation of agreement, correctness and acceptable errors. The closing operational advice partly repeats the calibration section and can be condensed.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-021", "D-022", "D-023", "D-038" ] }, { "cluster": "D", "article_file": "articles/benchmark-uncertainty-repeatability.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L31–33 joint counts yield A80/B85 and 15 discordances. L39 Wilson interval independently recomputes to 71.1–86.7%; L43 has the correct coverage interpretation. L49–55 preserves pairs and acknowledges clustered/small/degenerate-data limits. Primary checks: D-008, D-009, D-010.", "practical_usefulness": "The joint table, computed interval and explicit paired-bootstrap recipe bridge concepts to analysis. The draft distinguishes practical margins, selection/stopping and uncertainty units instead of asking readers to invent the central calculation.", "writing_quality": "The numerical progression is clear and appropriately qualified. Several caveats make the final sections dense; a small layout/copy edit would improve scanning.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-008", "D-009", "D-010", "D-039", "D-040" ] }, { "cluster": "D", "article_file": "articles/fair-ai-model-comparisons.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L33 scopes Sclar’s prompt-format findings; L49–53 explicitly refuses to equate nominal settings with internal computation and correctly accounts for reasoning. L75 accurately describes the shared harness. Primary checks: D-018, D-019, D-020.", "practical_usefulness": "The controls, adaptation/tuning budgets, context constraints and comparison modes give a followable protocol. Add one filled configuration row and an example of an API feature mismatch.", "writing_quality": "The controlled-versus-product distinction stays clear. Repeated freeze/log/disclose instructions could be consolidated into one record template.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-018", "D-019", "D-020" ] }, { "cluster": "D", "article_file": "articles/ai-latency-cost-per-successful-task.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21 separates streaming perception from completion; L41–55 handles categories, included reasoning and rendered-prefix caching correctly. L59–61 illustrative arithmetic (.008, .015, .0167) is correct and not presented as supplier pricing. Primary checks: D-001, D-002, D-003, D-004.", "practical_usefulness": "The timing boundaries, quantile method, censored timeouts, cache regime and all-attempt denominator make the cost/latency procedure implementable. The worked example keeps quality and cost separate.", "writing_quality": "Concrete accounting and timing terms are used consistently. The final checklist repeats earlier fields and can be shortened without losing the usable procedure.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-001", "D-002", "D-003", "D-004" ] }, { "cluster": "D", "article_file": "articles/evaluate-ai-agents-tools-outcomes.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L43–47 correctly distinguishes final-state reward from permission/policy evidence. L63’s ambiguous write recovery is contract-aware. L73–75 accurately defines task-averaged pass^k and warns against taking a power of the aggregate mean. Primary checks: D-033, D-034, D-035.", "practical_usefulness": "The return-record example combines a specific state target, forbidden refund, required confirmation, reset controls and lost-acknowledgement recovery. Ablations, failure categories and repeated-trial reporting give a usable system protocol.", "writing_quality": "Clear outcome/trajectory distinction and concrete failure examples. The report/checklist ending could be tightened; no substantive clarity defect blocks the method.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-033", "D-034", "D-035" ] }, { "cluster": "D", "article_file": "articles/maintain-llm-evaluation-suite.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L23 correctly cites continuous evaluation. L31–35/49–51 separates candidate changes from changed references/scoring, and L61–63 correctly avoids treating incident variants as representative independent traffic. Local cadence rules are recommendations. Primary checks: D-027, D-028, D-029.", "practical_usefulness": "Owners, triggers, layered suites and input/output/score triage form a usable maintenance routine. A filled bridge/changelog report would improve the interpretation step beyond the existing checklist.", "writing_quality": "The lifecycle is clear and the score-triage distinction is useful. Versioning and reporting cautions recur; consolidate those during a minor editorial pass.", "intent_and_metadata": "Front matter L2–8 supplies a specific reader query, accurate title/description and lowercase proposed slug; the corresponding D/cluster-map.json record agrees. The full draft fulfils that stated intent without invented search-volume claims." }, "publication_blockers": [], "fact_check_ids": [ "D-027", "D-028", "D-029" ] }, { "cluster": "E", "article_file": "articles/choose-a-metric-and-write-a-rubric.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L21/23 faithfully describes final-state and BFCL checks; L25 accurately distinguishes contextual similarity and correctness. L33’s automatic grounding proposal needs relationship/semantic validation: occurrence of a number or clause alone does not establish correct attribution. Primary checks: E-001, E-002, E-003, E-004, E-005.", "practical_usefulness": "L19–40 links checks to a concrete four-line Harbour Desk rubric and L48–58 covers format, length, alternative wording and path constraints. Include the promised scored draft anchors and a false-positive grounding example before treating the checker as sufficient.", "writing_quality": "The property-inbox example keeps the scoring discussion concrete. L11’s binary opening and L25’s blanket overlap dismissal are stronger than the nuanced advice that follows; soften those statements and tighten the repeated gate instructions.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-001", "E-002", "E-003", "E-004", "E-005" ] }, { "cluster": "E", "article_file": "articles/compare-models-on-the-same-terms.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L29’s 24-style sensitivity figures and L37–47’s dated prices/vendor cache ceilings match live primary material. L35 needs a model-specific definition of matched effort; L37 supplies prices/speed evidence but does not establish that serving queue alone changes answer quality. Primary checks: E-006, E-007, E-008, E-009, E-010, E-011.", "practical_usefulness": "The harness fields at L19–25, adapters, tuning budgets, caps, cache regimes and release-version distinction are practical. A completed comparison row would clarify how to match APIs with different controls; provider caches may not be explicitly clearable.", "writing_quality": "The matched-harness versus matched-budget distinction is clear and the examples are concrete. Categorical language about every setting and serving-mode quality needs qualification; some pricing detail can remain in the cost sibling.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-006", "E-007", "E-008", "E-009", "E-010", "E-011" ] }, { "cluster": "E", "article_file": "articles/design-evaluation-tasks-that-match-production.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L19/21/36–38/61 accurately attributes live vendor guidance and labels illustrative targets. L59 incorrectly permits removing a failed exposed item from the reported set without addressing failure-selection bias. L63’s number-presence suggestion is insufficient as a complete semantic grounding test. Primary checks: E-012, E-013, E-014, E-015, E-016.", "practical_usefulness": "The six-field task card, sampling slices, oversampling/reweighting and reference-solution checks at L23–55 are useful. The holdout instruction changes the reliability of the central procedure and needs substantive repair; add semantic false-positive tests for the checker.", "writing_quality": "The running task card makes the article easy to follow. L55’s assertion that a one-class test set will itself produce a one-class model is too causal unless linked to tuning; several categorical closing statements need softening.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [ "Materially incorrect holdout guidance: excluding failure-selected items and reporting the remaining set can inflate the score; disclose the historical full result and obtain a fresh independent test rather than treating the selected remainder as unbiased. Location: L59." ], "fact_check_ids": [ "E-012", "E-013", "E-014", "E-015", "E-016" ] }, { "cluster": "E", "article_file": "articles/evaluate-an-ai-model-for-your-job.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "HELM coverage/status at L19, documentation targets at L33, dated prices at L41 and Evals dates at L92 are supported. L33’s numeric-presence grounding sketch needs an explicit warning that matching figures does not validate their roles or all lease claims. Primary checks: E-017, E-018, E-019, E-020, E-021, E-022.", "practical_usefulness": "L27–43 defines a concrete fictional job, quality gates, budget and uncertainty; L55–80 supplies a coherent reading route and reusable record. A completed scorecard and semantic grounding counterexample would strengthen the method beyond lists.", "writing_quality": "Concrete examples and the three-instrument distinction make the guide clear. Some failure-mode paragraphs repeat details owned by the supporting articles, and repeated score-is-dated refrains could be shortened.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-017", "E-018", "E-019", "E-020", "E-021", "E-022" ] }, { "cluster": "E", "article_file": "articles/evaluate-the-agent-separately-from-the-model.md", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L23/25 correctly scopes current tau2 reward/release details, L45 attributes the git-history incident, and L49/53 accurately reports official documentation/vendor accounts. Under-half GPT-4o should explicitly say cross-domain aggregate, though it is not claimed to be current retail performance. Primary checks: E-023, E-024, E-025, E-026, E-027, E-028.", "practical_usefulness": "L17–47 gives vocabulary, state checks, separate tool/handoff errors, isolation and trial summaries. The Harbour Desk example is concrete, but a filled initial/goal state and permission assertion would make the system check executable.", "writing_quality": "The outcome-versus-transcript distinction is clear and the empty-outbox example is useful. L11’s one-call definition is too narrow as a universal model-benchmark definition, and some report requirements repeat earlier sections.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-023", "E-024", "E-025", "E-026", "E-027", "E-028" ] }, { "cluster": "E", "article_file": "articles/holdouts-contamination-and-leaked-answers.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "The time-sliced benchmark, failure-selected audit counts, July retraction, attributed Epoch conclusion and overlap study at L31–41 are supported. L49’s remaining-holdout reporting rule introduces selection bias and L53 overstates a published knowledge cutoff as certification that private later material is clean. Primary checks: E-029, E-030, E-031, E-032, E-033, E-034, E-035.", "practical_usefulness": "L47–55 offers concrete hashes, separate files, provenance and isolation rules. Its central failed-item deletion rule undermines the reported estimate; replace it with a fresh independent evaluation protocol and distinguish exposure risk from proof.", "writing_quality": "Mechanisms and public-audit chronology are readable, with careful attribution of reviewer conclusions. The terminology at L24 calling an unstated grader constraint reverse leakage is confusing; repeated audit detail overlaps the public-benchmark sibling.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [ "Materially incorrect holdout guidance: excluding failure-selected items and reporting the remaining set can inflate the score; disclose the historical full result and obtain a fresh independent test rather than treating the selected remainder as unbiased. Location: L49." ], "fact_check_ids": [ "E-029", "E-030", "E-031", "E-032", "E-033", "E-034", "E-035", "E-067" ] }, { "cluster": "E", "article_file": "articles/human-review-and-model-judges.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "L27 accurately scopes MT-Bench agreement/bias and G-Eval’s potential preference. L29’s current strong-judge recommendation and L49’s attributed CORE-Bench account match sources. The universal number-program claims at L21/43 need semantic validation and the calibration loop needs a fresh validation sample. Primary checks: E-036, E-037, E-038, E-039.", "practical_usefulness": "The seven-step calibration loop at L35–41 includes order, length and family tests with an explicitly non-powered starting count. Add independent validation after rubric repair, report ties/abstentions and show a scored draft before deploying the instrument.", "writing_quality": "Clear and concrete division of human, program and model responsibilities. The witness-with-an-interest analogy overstates what same-family origin alone establishes; several absolute program-versus-person sentences need a qualification.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-036", "E-037", "E-038", "E-039" ] }, { "cluster": "E", "article_file": "articles/keep-an-evaluation-honest-over-time.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "All specific shutdown fields, policies, five GPT-4 snapshots, October/June deprecation rows, Evals timeline, HELM status and promotional dates at L17–41 match live sources. L64’s range-overlap screening question should instead ask for a paired difference interval; overlap alone is not the test. Primary checks: E-040, E-041, E-042, E-043, E-044, E-045, E-046, E-047.", "practical_usefulness": "The dated calendar examples and change-trigger list at L45–52 make maintenance concrete. Nearly half the article is a provider retirement chronology; add a filled bridge/version report and assigned owner to make the operating procedure more durable.", "writing_quality": "The pin-and-retest logic is clear, and policy/table discrepancies are honestly preserved. The long vendor calendar interrupts the workflow; shorten or box it and reduce repeated souvenir/dated-object phrasing.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-040", "E-041", "E-042", "E-043", "E-044", "E-045", "E-046", "E-047" ] }, { "cluster": "E", "article_file": "articles/measure-latency-tokens-and-cost.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "Live primary pricing checks support the standard/cache/context/mode/residency/tool and promotional claims. The labelled call arithmetic at L39–51 is correct, but L41 calls the Luna/Astra rate difference fifty-fold when the worked mix gives100-fold; L31’s cache-write rates need the five-minute TTL qualification. Primary checks: E-048, E-049, E-050, E-051, E-052, E-053, E-054.", "practical_usefulness": "Detailed worked token, retry, search, long-reasoning and cost-per-success examples lead to a usable decision row. L57–63 distinguishes first/last token, throughput, concurrency and quality denominators. Explicitly handle p=0 in final checks.", "writing_quality": "The rate-card-versus-bill distinction is clear and the arithmetic is easy to follow. The extensive pricing prose is dense; a second table and shorter vendor paragraphs would improve scanning. Correct the numeric ratio wording.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-048", "E-049", "E-050", "E-051", "E-052", "E-053", "E-054" ] }, { "cluster": "E", "article_file": "articles/read-a-model-score-as-a-range.md", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "Normal/clustered/paired methods, trial metrics and rough interval calculations are supported. L19 categorically calls .87 versus .90 an undemonstrated loss without joint data; a valid paired counterexample has p=.03125. L44 correctly contradicts that shortcut. L33 needs explicit aggregate/retail scope. Primary checks: E-055, E-056, E-057, E-058, E-059, E-060.", "practical_usefulness": "The labelled table, correct binomial illustration and paired-analysis discussion are useful. The draft does not compute a paired interval or provide a recipe, despite promising a mechanical method, and the earlier significance shortcut can reverse the decision.", "writing_quality": "Generally clear and measured, with well-labelled invented data. The categorical L19 inference conflicts with L23/44; L56’s claim that a chart without intervals asserts zero width is rhetorical overstatement.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [ "Materially incorrect significance guidance at L19: a .87 rival can be significantly worse than .90 on matched200 items; marginal intervals alone cannot settle the paired comparison. Correct the shortcut and reconcile with L23/44." ], "fact_check_ids": [ "E-055", "E-056", "E-057", "E-058", "E-059", "E-060" ] }, { "cluster": "E", "article_file": "articles/when-a-public-benchmark-is-the-wrong-score.md", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 5 }, "evidence": { "accuracy_and_sources": "HELM, MMLU-Pro, the two dated OpenAI audits/retraction and maintainer scaffold distinctions are faithfully sourced. L26’s GPQA65/74/34 figures are abstract headlines; clarify they are not the selected448-item table, whose validator rates differ. Primary checks: E-061, E-062, E-063, E-064, E-065, E-066.", "practical_usefulness": "L19–24 gives bounded uses and L46–54 supplies a usable five-question screening procedure with a clear route to workload tests. Add a filled benchmark evidence card; the current case is mainly a conceptual science-versus-inbox contrast.", "writing_quality": "Concrete prose and explicit handling of inconsistent vendor baselines make the limitations readable. The lengthy coding-audit retelling overlaps the contamination article, and some universal selection claims at L52 need qualification.", "intent_and_metadata": "L2–8 metadata and E/cluster-map.json agree on ID, role, title, lowercase slug, description and a plausible specific practitioner query. The full article addresses that intent; no fabricated search-demand evidence appears." }, "publication_blockers": [], "fact_check_ids": [ "E-061", "E-062", "E-063", "E-064", "E-065", "E-066" ] } ] }