{ "finalised_at_utc": "2026-10-28T02:37:16.297199+00:00", "stage": "independent_scoring_before_pairwise", "rubric_version": "1.0", "rubric_sha256": "dccb6015970577c821a01750fa2353c75893b74be95a5d3f3a4fcea8835294cc", "anchors": { "0": "Absent or unusable", "1": "Major failures; extensive reconstruction required", "2": "Substantial corrections required", "3": "Useful with ordinary substantive editing", "4": "Strong; minor corrections required", "5": "Excellent against stated requirements; ready for final publication checks" }, "article_dimensions": [ "accuracy_and_sources", "practical_usefulness", "writing_quality", "intent_and_metadata" ], "articles": [ { "cluster": "A", "article_id": "01", "article_file": "A/articles/01-benchmarking-models.md", "article_sha256": "02ef1b0063811226630154bf8d00e5fec99dcad9a38efe7aab1d0cd9e6576364", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L23\u201335, 51: correctly scoped AgentBench, HELM, SWE-bench, exposure and judge citations; several broader methodological statements rely on editorial reasoning.", "practical_usefulness": "L31\u201377 and83\u201385: decision-first workflow, gates, full manifest and a bounded first-week plan; needs a consolidated reusable decision sheet.", "writing_quality": "Clear progression and defined model/application/agent boundaries; long paragraphs and repeated cautions atL75\u201385 could tighten.", "intent_and_metadata": "Title/query/descriptionL4\u20137 accurately promise defensible model/system selection; broad pillar intent is appropriate." }, "fact_check_ids": [ "A-001", "A-002", "A-003", "A-004", "A-005" ], "front_matter_evidence": "A/articles/01-benchmarking-models.md:2\u20138", "length_evidence": { "reviewer_body_words": 1935, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "02", "article_file": "A/articles/02-public-benchmarks.md", "article_sha256": "f5ad8eb31124edc1e51be9b96fdd9df9fceb92f5ce1d60001feed285301f090a", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L21, 31, 37: primary benchmark and contamination summaries check out; no current ranks or unsupported contamination accusation.", "practical_usefulness": "L47, 55\u201359: evidence record and concrete vendor questions; add a filled evidence-card example.", "writing_quality": "Plain-language explanation of denominators and variants; recurring warning passagesL39, 51, 53 could compress.", "intent_and_metadata": "L4\u20137 maps cleanly to leaderboard interpretation; metadata agrees with map." }, "fact_check_ids": [ "A-006", "A-007", "A-008" ], "front_matter_evidence": "A/articles/02-public-benchmarks.md:2\u20138", "length_evidence": { "reviewer_body_words": 1173, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "03", "article_file": "A/articles/03-task-design.md", "article_sha256": "2941c1dbfc59a64ce32563071fdbfd83e914f0944dc7c9f3d778a551a8af70f8", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L23 supports case variety; L43, 57 correctly distinguish weighting and related-case dependence; operational guidance is appropriately illustrative.", "practical_usefulness": "L31\u201335 case contract and L69\u201373 independent case review/lifecycle are actionable; coverage table is described rather than filled.", "writing_quality": "Good separation of source evidence from expected behaviour; final review section partly repeats pilot advice.", "intent_and_metadata": "Title and query address dataset construction; description accurately previews traceable cases." }, "fact_check_ids": [ "A-009", "A-010", "A-011" ], "front_matter_evidence": "A/articles/03-task-design.md:2\u20138", "length_evidence": { "reviewer_body_words": 1226, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "04", "article_file": "A/articles/04-datasets-holdouts.md", "article_sha256": "b0dae2d395c3ac6ff251dfef94e85d8aa694e3d903c1fcb577ff46f29f9c384c", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L19\u201327 carefully separate local leakage from opaque training exposure; Balloccu citation supports the latter.", "practical_usefulness": "L33\u201349 controls, overlap review and version bridge; L55\u201357 exposure response offers concrete actions; no executable split example.", "writing_quality": "Clear caveats about time splits and small groups; many short imperative paragraphs have similar rhythm.", "intent_and_metadata": "Specific holdout question; useful title and restrained description, matching map." }, "fact_check_ids": [ "A-012", "A-013", "A-014" ], "front_matter_evidence": "A/articles/04-datasets-holdouts.md:2\u20138", "length_evidence": { "reviewer_body_words": 1115, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "05", "article_file": "A/articles/05-metrics-rubrics.md", "article_sha256": "953744fc720a619266c44fd439f06d7f3c34914bf05be48526b125f4133731e6", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L21\u201329 correct denominators, precision/recall and validator limits; cited evaluation guidance supports task-specific scoring.", "practical_usefulness": "L33\u201343 observable rubric, gates and L61\u201363 boundary judgements; could add a numeric confusion-matrix example.", "writing_quality": "Clear distinctions among syntax, semantics and outcomes; repeats gate/aggregate warning across sections.", "intent_and_metadata": "L4\u20137 accurately promises metric selection and usable rubrics." }, "fact_check_ids": [ "A-015", "A-016", "A-017" ], "front_matter_evidence": "A/articles/05-metrics-rubrics.md:2\u20138", "length_evidence": { "reviewer_body_words": 1070, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "06", "article_file": "A/articles/06-human-model-judges.md", "article_sha256": "ed8b7ef2e9398086d0b37ef83c89d8dd408603597869654ede43608ee59a333f", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L31 real position-bias source; L33 limits neutrality claims; L39 imbalance warning sound. No universal judge reliability claimed.", "practical_usefulness": "L37\u201347 separates calibration/validation and audits false passes; L53\u201355 practical disagreement records; add a filled confusion matrix.", "writing_quality": "Accessible and careful; audit-budget ending repeats some earlier maintenance guidance.", "intent_and_metadata": "Specific human/model-judge intent; title slightly long but accurate." }, "fact_check_ids": [ "A-018", "A-019", "A-020" ], "front_matter_evidence": "A/articles/06-human-model-judges.md:2\u20138", "length_evidence": { "reviewer_body_words": 1082, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "07", "article_file": "A/articles/07-repeatability-uncertainty.md", "article_sha256": "f328e9e80ddecace7d1a9877792c89694d1ab05cadd0146c95b4b23ca59049b7", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L29 paired/group resampling and L31 binomial/coverage cautions are sound; source for wider sampling methods could be closer.", "practical_usefulness": "L27\u201343 usable paired protocol and decision margins; L15 numerical example stops before an actual interval calculation.", "writing_quality": "Precise, particularly non-significance vs equivalence; ending reiterates escalation rather than summarising a worked outcome.", "intent_and_metadata": "Query includes confidence intervals, which are explained conceptually; title and description fit." }, "fact_check_ids": [ "A-021", "A-022", "A-023" ], "front_matter_evidence": "A/articles/07-repeatability-uncertainty.md:2\u20138", "length_evidence": { "reviewer_body_words": 1062, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "08", "article_file": "A/articles/08-fair-comparisons.md", "article_sha256": "28cb9169a2ff153490af4fbe9094fff3e9074949d7b8803484cb038ac95197b0", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L19\u201323 attribution boundaries, L35 attempt budgets and L39 denominators sound; HELM is a precedent, not direct validation of local policy.", "practical_usefulness": "L27 run manifest and L57\u201359 capability-gap examples support implementation; a copyable manifest would improve use.", "writing_quality": "Controlled versus best-system distinction is lucid; similar manifest lists recur across series.", "intent_and_metadata": "Clear prompt/tool fairness question and accurate metadata." }, "fact_check_ids": [ "A-024", "A-025", "A-026" ], "front_matter_evidence": "A/articles/08-fair-comparisons.md:2\u20138", "length_evidence": { "reviewer_body_words": 1094, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "09", "article_file": "A/articles/09-latency-cost.md", "article_sha256": "52ee80ee7d0e9b994fbe052a26151ebf62663cba2ba76c795689afd7e09bf5aa", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L15 explicit invented rates; L29\u201331 arithmetic verified; L27 token categories accurate but no nearby official billing citation.", "practical_usefulness": "Worked per-attempt/per-success arithmetic plus tail diagnosisL55\u201357; define a complete sample scorecard to reduce reader assembly.", "writing_quality": "Clear units and hypothetical labels; minor token-count difference has no editorial consequence.", "intent_and_metadata": "Title/query/description match latency, usage and task economics." }, "fact_check_ids": [ "A-027", "A-028", "A-029" ], "front_matter_evidence": "A/articles/09-latency-cost.md:2\u20138", "length_evidence": { "reviewer_body_words": 1063, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "10", "article_file": "A/articles/10-agent-evaluation.md", "article_sha256": "7938293238f1ffcfc7c73ad3faf5ab0666360295ce4102e22c5609c4357d390c", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L15 AgentBench and L31 SWE-bench categories supported; L29 success claims correctly require external state.", "practical_usefulness": "L57\u201359 read failure and partial-write cases are useful; outcome/process gates and reset controls are concrete.", "writing_quality": "Good narrative from authority to trace to diagnosis; some generic cautions could become a compact trace example.", "intent_and_metadata": "Specific agent evaluation intent; title covers outcomes, traces and boundaries accurately." }, "fact_check_ids": [ "A-030", "A-031", "A-032" ], "front_matter_evidence": "A/articles/10-agent-evaluation.md:2\u20138", "length_evidence": { "reviewer_body_words": 1113, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "A", "article_id": "11", "article_file": "A/articles/11-results-maintenance.md", "article_sha256": "91976aa45f5dea837621c81ca869381a2aa0f6e78abb068d1c75fe7c65b0d6a5", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L41 continuous evaluation citation correct; L39 grouping and L45 scorer-version distinction sound; no inferred measured improvement.", "practical_usefulness": "L29\u201331 decision record, change triggers and L59 version-change review usable; reporting template remains prose.", "writing_quality": "Clear operational close to series; versioning and owners repeat earlier chapters but serve maintenance intent.", "intent_and_metadata": "Result interpretation and maintenance intent matched; proposed query is slightly awkward but plausible." }, "fact_check_ids": [ "A-033", "A-034", "A-035" ], "front_matter_evidence": "A/articles/11-results-maintenance.md:2\u20138", "length_evidence": { "reviewer_body_words": 1132, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-00", "article_file": "B/articles/evaluating-ai-models-and-agent-systems.md", "article_sha256": "483b4aa8565cb2d0149c7215248d70659ee4f053b20a97a16f55880c1fcf9667", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 3, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L22\u201336 many numerical results faithfully sourced; L90 hypothetical statistics recompute correctly. Some sweeping sample-size rhetoric needs qualification; journal year label requires correction.", "practical_usefulness": "Six-step loop and concrete planning ranges; L82\u201390 useful starting artefacts but counts must be decision-dependent.", "writing_quality": "Dense early catalogue of papers and repeated aphorisms; L106 overpromises that 100 tasks avoid most expensive mistakes.", "intent_and_metadata": "Broad production-evaluation query/title fit; metadata consistent with map." }, "fact_check_ids": [ "B-001", "B-002", "B-003", "B-004", "B-005", "B-006", "B-007", "B-008", "B-009", "B-010", "B-011", "B-012", "B-013", "B-068" ], "front_matter_evidence": "B/articles/evaluating-ai-models-and-agent-systems.md:2\u20138", "length_evidence": { "reviewer_body_words": 2089, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-01", "article_file": "B/articles/public-benchmarks-vs-workload-evaluation.md", "article_sha256": "942050ebe58533023b55a9c56ef022fe2a71dbcb30ecf94ae38adc98a51321f3", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Historical counts and Arena/BetterBench findings match originals; L58 cannot-estimate language is overly absolute about potentially matching distributions.", "practical_usefulness": "L66\u201384 two-stage process and shortlist table are directly useful; L90 catches-in-a-day claim is hypothetical but too assured.", "writing_quality": "Strong screening/selection structure; occasional slogan-like assertions and categorical phrasing need light revision.", "intent_and_metadata": "Distinct public-vs-local query, clear descriptive title and useful description." }, "fact_check_ids": [ "B-014", "B-015", "B-016", "B-017", "B-018", "B-019" ], "front_matter_evidence": "B/articles/public-benchmarks-vs-workload-evaluation.md:2\u20138", "length_evidence": { "reviewer_body_words": 1401, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-02", "article_file": "B/articles/designing-evaluation-tasks-and-datasets.md", "article_sha256": "f253c69f378c0f90ce15ffe58d202a65f1803fcc93a4d95411c2d0251dd9d14c", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 3, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Sources on cases, criteria drift and power are real; L111 sample ranges and L136/148 categorical prohibitions overstate what is necessary for valid evaluation.", "practical_usefulness": "L53\u201373 copyable task schema and L95\u2013100 labelling protocol; substantial practical content, with unsupported universal rules to soften.", "writing_quality": "Many useful lists but L14, 114, 142 dismiss small evaluations categorically; repetitive demo-versus-evidence framing needs substantive editing.", "intent_and_metadata": "Task/dataset title, query and metadata accurately identify the how-to purpose." }, "fact_check_ids": [ "B-020", "B-021", "B-022" ], "front_matter_evidence": "B/articles/designing-evaluation-tasks-and-datasets.md:2\u20138", "length_evidence": { "reviewer_body_words": 1402, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-03", "article_file": "B/articles/contamination-leakage-and-holdouts.md", "article_sha256": "d240b6f1966eb9563e47ec182f0a598251129cc915441b978535da73fb88745e", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L36\u201340 reported results correct; L44 calls revised May 2024 data initial release; L88 invents an upper-bound interpretation from nondisclosure.", "practical_usefulness": "Threat taxonomy and custody policy useful; L82\u201388 diagnostic conclusions can mislead, and quarterly/retirement rules L105\u2013106 need local rationale.", "writing_quality": "Mechanisms are clearly separated; emphatic headings and pharmaceutical analogy could tighten.", "intent_and_metadata": "Specific leakage/contamination intent and accurate overview description." }, "fact_check_ids": [ "B-023", "B-024", "B-025", "B-026", "B-027", "B-070", "B-071" ], "front_matter_evidence": "B/articles/contamination-leakage-and-holdouts.md:2\u20138", "length_evidence": { "reviewer_body_words": 1420, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-04", "article_file": "B/articles/objective-metrics-and-rubrics.md", "article_sha256": "8628a2a03ff3e72ac296319221fcf52158adcda7758043488419dcd6514fc6c5", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "pass@k formula and historical \u03c4-bench result correct; L44 token-F1 verbosity claim overbroad; L74 pooled score is traffic-like only with representative sampling.", "practical_usefulness": "Metric family mapping and L60\u201368 gated support rubric are usable; add n >= k to estimator and correct metric caveats.", "writing_quality": "Clear family-by-family reference structure; several categorical statements about strongest oracles need modest precision.", "intent_and_metadata": "Metric/rubric query fits; title and description preview the practical content." }, "fact_check_ids": [ "B-028", "B-029", "B-030", "B-031", "B-072", "B-073" ], "front_matter_evidence": "B/articles/objective-metrics-and-rubrics.md:2\u20138", "length_evidence": { "reviewer_body_words": 1407, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-05", "article_file": "B/articles/human-evaluation-and-model-judges.md", "article_sha256": "ceca67b4e76e89fb0e07fe9a16bcf902f0a75959c8635e08adf5f112c4fe22a0", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L72 >80% agreement and bias findings accurately scoped; L82 cancellation only holds in stated symmetric first-position case, not general immunity.", "practical_usefulness": "Kappa formula, study designs and fresh judge-validation sample L102\u2013108 are useful; richer false-pass accounting would improve protocol.", "writing_quality": "Readable instrument framing and sequence; broad claims about pairwise/reasoning reliability need contextual qualifiers.", "intent_and_metadata": "Specific judge reliability intent; descriptive title and metadata fit." }, "fact_check_ids": [ "B-032", "B-033", "B-034", "B-035", "B-074" ], "front_matter_evidence": "B/articles/human-evaluation-and-model-judges.md:2\u20138", "length_evidence": { "reviewer_body_words": 1314, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-06", "article_file": "B/articles/repeatability-variance-and-uncertainty.md", "article_sha256": "545cc8a5cf8e7f530c81c1ef7f08d15db728fe12686f9ae35010783e20d4716a", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Core formulas, hypothetical 3.9 pp / 900-case calculation, Card 75% and Miller 3x / 969 all traceable; normal intervals need stronger small or rare-failure sample limits.", "practical_usefulness": "Equations, paired numerical example, power calculation and report table provide implementable depth; distinguish trial-mean variance from binary variance in table.", "writing_quality": "Good explanations of pairing/clusters; repeated \u201cnot a 70% system\u201d wording oversimplifies differing reliability estimands.", "intent_and_metadata": "Statistical-significance query met with calculations rather than generic warnings." }, "fact_check_ids": [ "B-036", "B-037", "B-038", "B-039", "B-040", "B-041", "B-042", "B-043", "B-044", "B-069" ], "front_matter_evidence": "B/articles/repeatability-variance-and-uncertainty.md:2\u20138", "length_evidence": { "reviewer_body_words": 1439, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-07", "article_file": "B/articles/prompt-tool-and-configuration-fairness.md", "article_sha256": "6b8de247ea51100cb2414eddee8dfe7ea9f78216f49f3f8ac3862cd751b7734c", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L110 always-increases assertion unsupported; L114 same nominal effort is not a cross-model compute match; current Bash Only description and Inspect source are correct.", "practical_usefulness": "Manifest L41\u201372 and fairness audit useful; repair effort equivalence and distinguish operating constraints before use.", "writing_quality": "Clear controlled/optimised prompt explanation; absolute opening is moderated later, but effort section remains misleading.", "intent_and_metadata": "Precise prompt/tool fairness query and descriptive title agree with content." }, "fact_check_ids": [ "B-045", "B-046", "B-047", "B-075", "B-076" ], "front_matter_evidence": "B/articles/prompt-tool-and-configuration-fairness.md:2\u20138", "length_evidence": { "reviewer_body_words": 1295, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-08", "article_file": "B/articles/latency-token-usage-and-cost.md", "article_sha256": "6b9f08bd953ed25a43732e00dda133ad4b30c517b540c28ae9f897c2e6ab2d1b", "scores": { "accuracy_and_sources": 3, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Most dated rates/schedules and arithmetic verified. L75 omits the Haiku >100K fivefold price tier; L44/53 visible-output definition risks excluding billed reasoning.", "practical_usefulness": "Cost worksheet, incremental economics and frontier helpful; billing formula and tier table must be repaired before budgeting.", "writing_quality": "Readable worked arithmetic and explicit hypothetical labels; separate literal rate assumptions from total billed usage more clearly.", "intent_and_metadata": "Operational cost/latency intent clear; dated price table supports comparison but adds maintenance obligation." }, "fact_check_ids": [ "B-048", "B-049", "B-050", "B-051", "B-052", "B-053", "B-054", "B-055", "B-056", "B-077", "B-078", "B-079", "B-080" ], "front_matter_evidence": "B/articles/latency-token-usage-and-cost.md:2\u20138", "length_evidence": { "reviewer_body_words": 1348, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-09", "article_file": "B/articles/evaluating-agents-and-tool-use.md", "article_sha256": "6542939325f2b0d76e88cd06742028e7e39b4dc02291e6db8e5514303d37df87", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "SWE-bench, GAIA, \u03c4-bench counts/results match originals; LiveCodeBench design accurately described; pass^k is a specific reliability estimand.", "practical_usefulness": "Outcome/trajectory taxonomy, reset protocol and ablation checklist useful; failure-injection assertions are less worked than the protocol lists suggest.", "writing_quality": "Clear system decomposition; public benchmark catalogue consumes space and overlaps pillar.", "intent_and_metadata": "Agent/tool-use query and boundary-focused title fit accurately." }, "fact_check_ids": [ "B-057", "B-058", "B-059", "B-060", "B-061", "B-062", "B-063", "B-064" ], "front_matter_evidence": "B/articles/evaluating-agents-and-tool-use.md:2\u20138", "length_evidence": { "reviewer_body_words": 1423, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "B", "article_id": "ai-eval-10", "article_file": "B/articles/interpreting-results-and-maintaining-evals.md", "article_sha256": "439aa8110bfecd429b038bb2770709bb9c30623d0e841005c6c64bad2f2daac9", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Current Evals 31 October / 30 November notice verified; BetterBench / The Leaderboard Illusion attribution accurate; 90 runs are not automatically 90 pairwise statistical comparisons.", "practical_usefulness": "Memo template and maturity/cadence framework practical; holdout retirement wording conflicts slightly with the two-decision rule in leakage article.", "writing_quality": "Useful sequence and concrete deprecation example; statistical significance L41 is too shorthand and headings repeat familiar cautions.", "intent_and_metadata": "Interpretation/maintenance query and title correctly target post-run decisions." }, "fact_check_ids": [ "B-065", "B-066", "B-067" ], "front_matter_evidence": "B/articles/interpreting-results-and-maintaining-evals.md:2\u20138", "length_evidence": { "reviewer_body_words": 1342, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "P01", "article_file": "C/articles/ai-model-benchmarking-guide.md", "article_sha256": "fd7e9103a264abe872bf0c5c3982a41618ad5d06fca5c732d0e6543a215ddb6d", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L33 MMLU, L73 official guidance and L91 dependence match the sources; L105 arithmetic verified. Claims consistently distinguish evidence and hypothetical examples.", "practical_usefulness": "L123 names six concrete artefacts and L119 gives review triggers. The workflow is actionable; a filled decision contract would reduce reader assembly.", "writing_quality": "Clear progression with a sustained support-assistant example. Recurring formal language about contracts and evidence could be lighter.", "intent_and_metadata": "The broad pillar query, title and description accurately frame an evidence-based model choice." }, "fact_check_ids": [ "C-001", "C-002", "C-003", "C-035" ], "front_matter_evidence": "C/articles/ai-model-benchmarking-guide.md:2\u20138", "length_evidence": { "reviewer_body_words": 1943, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S01", "article_file": "C/articles/read-ai-benchmark-leaderboards.md", "article_sha256": "491e407fc2d7ea0753e2c3d0f014d118b44ba4f3f544ec63cf2fb7b3c3a72603", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "HumanEval attempt metrics, HELM and the dated Bash Only description verified. No unsupported rank, contamination accusation or interval inference.", "practical_usefulness": "L79 evidence-card fields and the local follow-up test make a useful audit procedure. A filled card would strengthen the example.", "writing_quality": "Good progression through task, score and configuration. Repeated references to the evidence card could tighten.", "intent_and_metadata": "Specific leaderboard-reading query and clear title; the description states useful boundaries." }, "fact_check_ids": [ "C-004", "C-005", "C-006", "C-007" ], "front_matter_evidence": "C/articles/read-ai-benchmark-leaderboards.md:2\u20138", "length_evidence": { "reviewer_body_words": 1167, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S02", "article_file": "C/articles/design-llm-evaluation-tasks.md", "article_sha256": "202fdb19037d4dc5d0ad310ee831e6f57d6c67c24a824ad39608bf57cee914f3", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L55 CheckList matrix accurately sourced. Future-information boundaries and challenge-versus-traffic distinctions are sound.", "practical_usefulness": "RETURN-014, permitted outcomes, coverage axes and the observation point are concrete. An actual record or matrix would make the schema easier to reuse.", "writing_quality": "Precise treatment of clarification and curator ambiguity. The mainly prose format provides limited scanning relief.", "intent_and_metadata": "Task-design intent is accurately addressed; the title and description distinguish this article from dataset protection." }, "fact_check_ids": [ "C-008", "C-009", "C-010" ], "front_matter_evidence": "C/articles/design-llm-evaluation-tasks.md:2\u20138", "length_evidence": { "reviewer_body_words": 1086, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S03", "article_file": "C/articles/evaluation-dataset-leakage-holdouts.md", "article_sha256": "0ba74143e7e830b9d0e68f90d7cb88cfa8f00df75f5f89572526dc9e328a0337", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Leakage, Datasheets, Shi and Oren associations verified. Canonical-order detection retains its exchangeability and access limitations.", "practical_usefulness": "L29 provenance, L49 access, L59 overlap dispositions and version corrections are actionable. A worked grouped split would improve depth.", "writing_quality": "Clear separation of leakage, contamination and mismatch. Some lifecycle advice repeats the maintenance article.", "intent_and_metadata": "Dataset-leakage query and accurate description align with the distinct data-protection purpose." }, "fact_check_ids": [ "C-011", "C-012", "C-013", "C-014" ], "front_matter_evidence": "C/articles/evaluation-dataset-leakage-holdouts.md:2\u20138", "length_evidence": { "reviewer_body_words": 1123, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S04", "article_file": "C/articles/llm-evaluation-metrics-rubrics.md", "article_sha256": "435e5620e7c47e8b983593c2ccf373c2a7ee7712cb5cd0c646766353354b420e", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Precision, recall, F1, accuracy and the always-negative baseline recomputed. Macro averaging and timeout-denominator guidance are sound.", "practical_usefulness": "L39\u201345 worked confusion matrix, L51\u201355 rubric anchors and L61\u201367 critical flags enable immediate application.", "writing_quality": "Concrete calculations improve clarity. Closing cross-links are dense but useful.", "intent_and_metadata": "The metric query, title and description match the formulas, rubrics and release gates." }, "fact_check_ids": [ "C-015", "C-016", "C-036", "C-037" ], "front_matter_evidence": "C/articles/llm-evaluation-metrics-rubrics.md:2\u20138", "length_evidence": { "reviewer_body_words": 1070, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S05", "article_file": "C/articles/human-reviewers-llm-judges.md", "article_sha256": "c1b22518bd172a0b8ccc6d11252929daf960e0c1abb19b0e277ee6fb52e32058", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Bias claims accurately reference MT-Bench. L61 always-pass example recomputed; no invented local agreement rate.", "practical_usefulness": "Separates calibration from fresh validation, handles invalid verdicts and samples automated passes. A filled adjudication worksheet would improve adoption.", "writing_quality": "Good distinction between evaluator uncertainty and candidate clarification. Uniform paragraph patterns could tighten.", "intent_and_metadata": "The human/model-grading query, title and description accurately target trustworthy judging." }, "fact_check_ids": [ "C-017", "C-018", "C-038" ], "front_matter_evidence": "C/articles/human-reviewers-llm-judges.md:2\u20138", "length_evidence": { "reviewer_body_words": 1120, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S06", "article_file": "C/articles/benchmark-uncertainty-repeatability.md", "article_sha256": "685adad4d72eaa8a13f604361ed8de2fe07195e95f459e040672d4e88f25af68", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L31 paired counts, L39 Wilson interval and L43 coverage interpretation verified. Paired resampling and small-sample caveats are accurate.", "practical_usefulness": "A calculated interval, paired counts, a 10, 000-resample recipe and explicit group weighting provide actionable technical depth.", "writing_quality": "Clear limits on inference and no invented bootstrap result. A displayed formula or table would make calculations easier to scan.", "intent_and_metadata": "The improvement question and significance query are met; the description accurately promises paired analysis." }, "fact_check_ids": [ "C-019", "C-020", "C-021", "C-039", "C-040" ], "front_matter_evidence": "C/articles/benchmark-uncertainty-repeatability.md:2\u20138", "length_evidence": { "reviewer_body_words": 1067, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S07", "article_file": "C/articles/fair-ai-model-comparisons.md", "article_sha256": "8ee2e078f84c7825b371e2c0661d50fa65e50d7699a4f2c1179a740342e9eda9", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Sclar format sensitivity, reasoning-token billing and the harness source verified. Nominal controls are never claimed to equal internal computation.", "practical_usefulness": "L49\u201355 unsupported settings and context adaptations, L69\u201373 manifest fields and sensitivity checks address practical fairness failures.", "writing_quality": "Strong controlled-versus-deployable explanation. The long manifest paragraph could become a checklist.", "intent_and_metadata": "The fair-comparison query and title are specific; the description accurately states configuration boundaries." }, "fact_check_ids": [ "C-022", "C-023", "C-024" ], "front_matter_evidence": "C/articles/fair-ai-model-comparisons.md:2\u20138", "length_evidence": { "reviewer_body_words": 1094, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S08", "article_file": "C/articles/ai-latency-cost-per-successful-task.md", "article_sha256": "44c0b48b068df4c97043ddddd7756c3727ceabe33da4725c3e482cc5eee0adb5", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Streaming, pricing categories and prefix rules match official documentation. Both separate hypothetical calculations verified.", "practical_usefulness": "L33 timeout censoring, L43 avoidance of double counting, L65 zero-success handling and L79 reconciliation make accounting robust.", "writing_quality": "Clear units and explicitly separate examples. A worksheet would make the cost section easier to scan.", "intent_and_metadata": "The cost-per-success title and latency query match reader needs; the description includes retries, caching and failures." }, "fact_check_ids": [ "C-025", "C-026", "C-027", "C-041", "C-042" ], "front_matter_evidence": "C/articles/ai-latency-cost-per-successful-task.md:2\u20138", "length_evidence": { "reviewer_body_words": 1106, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S09", "article_file": "C/articles/evaluate-ai-agents-tools-outcomes.md", "article_sha256": "dd328193e6c353864ad379af6f96755d6b6f291f7828d58fbbd75876f12e53ff", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 5, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "\u03c4-bench state and metric definitions verified. The paper itself notes that correct final state can miss permission violations; no aggregate-power shortcut is recommended.", "practical_usefulness": "L41 exactly-one-record predicate, L61 lost acknowledgement and L63 reconciliation before retry provide concrete recovery and permission tests.", "writing_quality": "Clear observable behaviour without treating a trace narrative as proof. Some short sections could consolidate.", "intent_and_metadata": "The agent query, title and description accurately reflect outcomes, actions and recovery." }, "fact_check_ids": [ "C-028", "C-029", "C-030", "C-031" ], "front_matter_evidence": "C/articles/evaluate-ai-agents-tools-outcomes.md:2\u20138", "length_evidence": { "reviewer_body_words": 1098, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "C", "article_id": "S10", "article_file": "C/articles/maintain-llm-evaluation-suite.md", "article_sha256": "19a310762169cfa03ea1be8ad4813de9f4214dd46d0c92f4b0c648d7acc30ee1", "scores": { "accuracy_and_sources": 5, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "The continuous-evaluation citation is correct. Grader, task and version changes remain distinct; incident-sampling limits are retained.", "practical_usefulness": "L49\u201355 input/output/score triage and L61 date-convention incident are useful. A copyable change record would strengthen implementation.", "writing_quality": "Focused operational flow. References to owners and versions recur frequently.", "intent_and_metadata": "The continuous-evaluation query and maintenance title match the scope; metadata is accurate." }, "fact_check_ids": [ "C-032", "C-033", "C-034" ], "front_matter_evidence": "C/articles/maintain-llm-evaluation-suite.md:2\u20138", "length_evidence": { "reviewer_body_words": 1088, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "model-benchmarking", "article_file": "D/articles/choosing-model-benchmark.md", "article_sha256": "fef48867b15cb05e0ee821db08878e86d7262c988e47db537a3139a94cdf37f1", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L33 MMLU/HELM, L76 pricing categories and L84 SWE-bench verified. Example thresholds are explicitly illustrative.", "practical_usefulness": "Concrete invoice fields, latency/budget conditions and L88 decision rule. A consolidated run plan would improve practical handoff.", "writing_quality": "Accessible framework with clear hard gates. Long paragraphs and repeated cautions could tighten.", "intent_and_metadata": "The workload-specific pillar intent is accurate; metadata agrees with the map." }, "fact_check_ids": [ "D-001", "D-002", "D-003", "D-004" ], "front_matter_evidence": "D/articles/choosing-model-benchmark.md:2\u20138", "length_evidence": { "reviewer_body_words": 2069, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "public-benchmarks", "article_file": "D/articles/public-benchmark-literacy.md", "article_sha256": "a0282ef1afe7813e3e8bbdc2fe8f035a4eeb8e981ad58ac3f61bf2e1cfc0e483", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "MMLU, HELM and contamination claims are sound. L39 attaches the wrong author, Zheng, to a real position-bias paper.", "practical_usefulness": "L55 evidence-table fields and L63\u201372 checklist are usable; a completed benchmark card is missing.", "writing_quality": "Clear bounded interpretation. Rhetorical questions sometimes repeat the pillar warning.", "intent_and_metadata": "The title/query and description are accurate. Normalise case in the proposed slug before CMS use." }, "fact_check_ids": [ "D-005", "D-006", "D-007", "D-008", "D-037" ], "front_matter_evidence": "D/articles/public-benchmark-literacy.md:2\u20138", "length_evidence": { "reviewer_body_words": 1111, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "task-evaluation", "article_file": "D/articles/build-task-eval.md", "article_sha256": "462ad4bfe322b9b72c5a567c70147fa55dd4a8851ac0f700ddf74021f5518536", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Synthetic-data prevalence, incident grouping and holdout-reuse guidance are sound. Most methodological claims lack a nearby citation.", "practical_usefulness": "L19 task contract and L71\u201373 ticket dataset/label repair are concrete. Showing an actual case row would improve reuse.", "writing_quality": "A well-structured contract and useful pitfalls, with some overlap with the dataset article.", "intent_and_metadata": "Task-design intent is clear; the description accurately covers labels and splits." }, "fact_check_ids": [ "D-009", "D-010", "D-011" ], "front_matter_evidence": "D/articles/build-task-eval.md:2\u20138", "length_evidence": { "reviewer_body_words": 1392, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "dataset-integrity", "article_file": "D/articles/datasets-holdouts-contamination.md", "article_sha256": "d3884cddf2731c217e0464cefafbe103f80a65fd11ca21656d5b9ddf5cb39188", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Yang and the 2026 Xiao/Cheng papers are real. The ranking conclusion is appropriately study-specific; broader guidance has sparse nearby sources.", "practical_usefulness": "Three partitions, access checklist and version bridge are usable. Overlap remediation remains procedural rather than worked.", "writing_quality": "Good freshness-versus-label-quality tradeoff. Introductory generalisations can compress.", "intent_and_metadata": "The leakage query, title and description match the data-integrity focus." }, "fact_check_ids": [ "D-012", "D-013", "D-014" ], "front_matter_evidence": "D/articles/datasets-holdouts-contamination.md:2\u20138", "length_evidence": { "reviewer_body_words": 1178, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "evaluation-metrics", "article_file": "D/articles/metrics-rubrics.md", "article_sha256": "34346ae556e7ce5c4655dcb5bf1db8d35f87529e3532fd9b8cd1625d58200e29", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Definitions, imbalance example and macro/weighted qualifications are correct. Official grader descriptions verified.", "practical_usefulness": "Groundedness anchors, invoice scorecard and selective coverage are useful; no full numerical confusion matrix.", "writing_quality": "Precise ordinal-scale and abstention discussion. Several lists could consolidate.", "intent_and_metadata": "The metric/rubric query is met; the title and description accurately promise outcome alignment." }, "fact_check_ids": [ "D-015", "D-016", "D-017", "D-018" ], "front_matter_evidence": "D/articles/metrics-rubrics.md:2\u20138", "length_evidence": { "reviewer_body_words": 1335, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "human-judging", "article_file": "D/articles/human-evaluation-model-judges.md", "article_sha256": "662de703e8aa92a76fcff333ba9d944a174088425f2ddd6f1851387b5a59d9d6", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "L35 mislabels the real position-bias paper as Zheng et al. The substantive claim is supported; other judge guidance is appropriately conditional.", "practical_usefulness": "Seven-step validation, policy grounding and audits of passes are practical. Emphasise separate fresh validation more clearly.", "writing_quality": "Readable hybrid-review argument with useful uncertainty marking; the bias list repeats some metrics material.", "intent_and_metadata": "The human/model-judge intent is accurate, with a useful descriptive title." }, "fact_check_ids": [ "D-019", "D-020", "D-021", "D-038" ], "front_matter_evidence": "D/articles/human-evaluation-model-judges.md:2\u20138", "length_evidence": { "reviewer_body_words": 1247, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "repeatability", "article_file": "D/articles/repeatability-uncertainty.md", "article_sha256": "39d0eaa12cd02f0e39f4c36269f91f498c38cc2e591f0eeae85d9f2faad78143", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 3, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Bouthillier is a real variance precedent. Dependence, paired intervals and non-overlap interpretation are sound.", "practical_usefulness": "L64\u201373 protocol is useful, but the reader never calculates an interval or applied test. L77 remains an intuitive inconclusive example; ordinary substantive expansion is needed for the technical promise.", "writing_quality": "Clear variation taxonomy without false precision. The main technical promise outpaces worked depth.", "intent_and_metadata": "The confidence-interval query is relevant, but the description would be better fulfilled by an actual calculation. Normalise slug case." }, "fact_check_ids": [ "D-022", "D-023", "D-024" ], "front_matter_evidence": "D/articles/repeatability-uncertainty.md:2\u20138", "length_evidence": { "reviewer_body_words": 1284, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "fair-comparison", "article_file": "D/articles/fair-model-comparisons.md", "article_sha256": "2e72884a4769384dda73dcc24b39fe138de9b23d4eca6cf78054adc7d50187fd", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Controlled/system attribution, budgets and unsupported-control boundaries are sound. Most advice is uncited editorial recommendation.", "practical_usefulness": "Context, tool, parser, safety and tuning controls plus L51\u201353 experiments are usable. The configuration ledger is described in prose.", "writing_quality": "Clear nuance about asymmetric APIs and observable outputs. The generic closing repeats the opening.", "intent_and_metadata": "The fair-comparison query, title and description are specific and accurate." }, "fact_check_ids": [ "D-025", "D-026", "D-027" ], "front_matter_evidence": "D/articles/fair-model-comparisons.md:2\u20138", "length_evidence": { "reviewer_body_words": 1134, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "latency-cost", "article_file": "D/articles/latency-cost.md", "article_sha256": "ad86e020157896495428016814686f7d9ca1a8df0fddb0a209852ba25e22007d", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Official billing categories verified. L57 totals $258/$127 recomputed; no fixed provider rates claimed.", "practical_usefulness": "The human-review worksheet and matched-quality caveat are strong. L27 must define base charges carefully so adding retry charges cannot double count them.", "writing_quality": "Clear human-versus-API economics. Finish the cost-per-success example with an explicit accepted-task denominator.", "intent_and_metadata": "The cost/latency query is met; normalise uppercase letters in the proposed slug." }, "fact_check_ids": [ "D-028", "D-029", "D-036" ], "front_matter_evidence": "D/articles/latency-cost.md:2\u20138", "length_evidence": { "reviewer_body_words": 1095, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "agent-evaluation", "article_file": "D/articles/agent-system-evaluation.md", "article_sha256": "595c30702190e0616b5582482ddbf0c64f1f9681e38552256aefcf69c010cd34", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "The trace guide and 2026 Pro Verified preprint are real. Claims are appropriately specific; scaffold attribution is sound.", "practical_usefulness": "Pinned coding environment, external acceptance checks and ablation are useful. Recovery is listed without a filled failure-injection trace.", "writing_quality": "Strong model/system boundary. Generic warnings recur and the final paragraph has dense cross-links.", "intent_and_metadata": "The agent/tool query and description are accurate; normalise proposed slug case." }, "fact_check_ids": [ "D-030", "D-031", "D-032" ], "front_matter_evidence": "D/articles/agent-system-evaluation.md:2\u20138", "length_evidence": { "reviewer_body_words": 1335, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." }, { "cluster": "D", "article_id": "evaluation-maintenance", "article_file": "D/articles/maintaining-evaluations.md", "article_sha256": "a362430963b8df0905d9b53cd01478fa3e034453f2c7db19ae74701f5334b2e3", "scores": { "accuracy_and_sources": 4, "practical_usefulness": 4, "writing_quality": 4, "intent_and_metadata": 4 }, "evidence": { "accuracy_and_sources": "Continuous-evaluation guidance verified. Anchor/new sets and grader versions stay distinct; no measured cadence is claimed.", "practical_usefulness": "L56 illustrative suite cadence, owner roles and bridges are usable. A completed changelog or decision record would improve reuse.", "writing_quality": "Clear continuity/staleness tradeoff. Version/checklist language overlaps the pillar.", "intent_and_metadata": "The maintenance query, title and description fit; normalise proposed slug case." }, "fact_check_ids": [ "D-033", "D-034", "D-035" ], "front_matter_evidence": "D/articles/maintaining-evaluations.md:2\u20138", "length_evidence": { "reviewer_body_words": 1067, "within_range": true }, "anchor_application": "Integer anchors from unchanged rubric.json. A4 means strong with minor corrections; a3 needs ordinary substantive editing; a5 is excellent for this dimension, pending final publication checks." } ], "assessment_limits": "Integer editorial anchors are judgements,not measurements ofmodelability. Fullreading is distinct fromsampledfactverification. No cross-candidate preferenceused toadjustscores." }