{ "scoring_stage": "independent_before_pairwise", "scored_at_utc": "2026-10-25T14:40:05.238744+00:00", "review_local_calendar_date": "2026-10-26", "rubric_version": "1.0", "rubric_sha256": "dccb6015970577c821a01750fa2353c75893b74be95a5d3f3a4fcea8835294cc", "anchor_policy": "Integer 0–5 anchors from unchanged rubric; evidence for each of four article dimensions. Scores are editorial judgements, not measured model performance.", "anchors": { "0": "Absent or unusable", "1": "Major failures; extensive reconstruction required", "2": "Substantial corrections required", "3": "Useful with ordinary substantive editing", "4": "Strong; minor corrections required", "5": "Excellent against stated requirements; ready for final publication checks" }, "articles": [ { "cluster": "A", "article_id": "model-benchmarking", "article_file": "A/articles/choosing-model-benchmark.md", "full_draft_read": true, "review_body_words": 2052, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "MMLU/HELM/SWE-bench descriptions and dated pricing categories check out; examples explicitly hypothetical.", "claim_ids": [ "A-model-benchmarking-01", "A-model-benchmarking-02", "A-model-benchmarking-03", "A-model-benchmarking-04", "A-model-benchmarking-05" ] }, "practical_usefulness": { "score": 4, "evidence": "Invoice extraction decision contract, hard gates and controlled versus best-configuration designs make the overview usable.", "article_file": "A/articles/choosing-model-benchmark.md" }, "writing_quality": { "score": 4, "evidence": "Clear sequence from decision to maintenance, with ten contextual handoffs; some repeated cautions and a long final recap.", "article_file": "A/articles/choosing-model-benchmark.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: How to benchmark AI models for your own workload. Query: AI model benchmarking for real workloads. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Add a compact completed decision-record example and trim repeated warnings.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "public-benchmarks", "article_file": "A/articles/public-benchmark-literacy.md", "full_draft_read": true, "review_body_words": 1095, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Scope claims verified; position-bias source is real but labelled Zheng rather than Shi; paraphrase-audit description is loose.", "claim_ids": [ "A-public-benchmarks-01", "A-public-benchmarks-02", "A-public-benchmarks-03", "A-public-benchmarks-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Eight-question reading checklist and evidence-table fields bridge leaderboard interpretation to a local shortlist.", "article_file": "A/articles/public-benchmark-literacy.md" }, "writing_quality": { "score": 4, "evidence": "Focused, readable and restrained; contamination and uncertainty discussion repeat later articles.", "article_file": "A/articles/public-benchmark-literacy.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: How to read public AI model benchmarks without overreading them. Query: public AI model benchmarks. Description matches the draft’s reader question; no search volumes asserted. Uppercase slug segments need routine URL-style normalisation. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Correct the author label and describe the contamination study’s actual method.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "task-evaluation", "article_file": "A/articles/build-task-eval.md", "full_draft_read": true, "review_body_words": 1387, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Grouping, leakage and designed-challenge advice accord with checked references; article itself has no nearby external citations.", "claim_ids": [ "A-task-evaluation-01", "A-task-evaluation-02", "A-task-evaluation-03" ] }, "practical_usefulness": { "score": 4, "evidence": "Five-part task contract and 120-case routing example explain ambiguity, slice weighting and split roles.", "article_file": "A/articles/build-task-eval.md" }, "writing_quality": { "score": 4, "evidence": "Concrete ticket-routing narrative with clear headings and checklist; several data-lifecycle points recur in the dataset article.", "article_file": "A/articles/build-task-eval.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: How to design an AI evaluation around a real task. Query: design an AI model evaluation. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Add a filled case row and nearby support for grouped splits/calibration.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "dataset-integrity", "article_file": "A/articles/datasets-holdouts-contamination.md", "full_draft_read": true, "review_body_words": 1156, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Xiao/Cheng preprint and caveat verified; Yang attribution needs a more exact description of injected variants and semantic filtering.", "claim_ids": [ "A-dataset-integrity-01", "A-dataset-integrity-02", "A-dataset-integrity-03", "A-dataset-integrity-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Eight-step contamination audit, answer-key separation, provenance and version bridges are operationally useful.", "article_file": "A/articles/datasets-holdouts-contamination.md" }, "writing_quality": { "score": 4, "evidence": "Cautious about closed training exposure; three-part holdout structure is accessible, though generalisation discussion is wordy.", "article_file": "A/articles/datasets-holdouts-contamination.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Keep your AI benchmark honest: datasets, holdouts and leakage. Query: AI benchmark data leakage. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Correct Yang method summary; include an example overlap disposition.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "evaluation-metrics", "article_file": "A/articles/metrics-rubrics.md", "full_draft_read": true, "review_body_words": 1329, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Precision/recall, macro weighting and grader-ordering claims verified; source-supported recommendations are sensibly bounded.", "claim_ids": [ "A-evaluation-metrics-01", "A-evaluation-metrics-02", "A-evaluation-metrics-03", "A-evaluation-metrics-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Anchored groundedness scale, hard gates and invoice scorecard are useful; no worked confusion matrix or completed aggregation example.", "article_file": "A/articles/metrics-rubrics.md" }, "writing_quality": { "score": 4, "evidence": "Clear syntax-versus-semantics and abstention explanation; long inventories could be tighter.", "article_file": "A/articles/metrics-rubrics.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Choose metrics and rubrics that reward the right AI behaviour. Query: AI evaluation metrics and rubrics. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Work through one complete metric calculation and boundary-case rubric score.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "human-judging", "article_file": "A/articles/human-evaluation-model-judges.md", "full_draft_read": true, "review_body_words": 1233, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Human calibration and grader-gaming caveats supported; order-bias citation has wrong author label.", "claim_ids": [ "A-human-judging-01", "A-human-judging-02", "A-human-judging-03", "A-human-judging-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Seven-step model-judge validation and audit of apparent passes give a usable hybrid workflow.", "article_file": "A/articles/human-evaluation-model-judges.md" }, "writing_quality": { "score": 4, "evidence": "Explains disagreement causes and uncertainty well; agreement measures remain a broad survey.", "article_file": "A/articles/human-evaluation-model-judges.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: When humans and model judges should score AI outputs. Query: human evaluation and LLM as a judge. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Correct Shi attribution; show a small false-pass audit table.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "repeatability", "article_file": "A/articles/repeatability-uncertainty.md", "full_draft_read": true, "review_body_words": 1272, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Variance source, paired-difference principle and dependence warning supported; numeric example clearly invented.", "claim_ids": [ "A-repeatability-01", "A-repeatability-02", "A-repeatability-03", "A-repeatability-04" ] }, "practical_usefulness": { "score": 3, "evidence": "Distinguishes new cases from repeats and gives an eight-step protocol, but does not show how to compute the proposed interval.", "article_file": "A/articles/repeatability-uncertainty.md" }, "writing_quality": { "score": 4, "evidence": "Plain-language statistical cautions are clear; many instructions defer the actual method choice.", "article_file": "A/articles/repeatability-uncertainty.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Repeatability and uncertainty in AI model evaluations. Query: AI benchmark confidence intervals and repeatability. Description matches the draft’s reader question; no search volumes asserted. Uppercase slug segments need routine URL-style normalisation. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Add a worked paired difference or explicit bootstrap recipe with assumptions.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "fair-comparison", "article_file": "A/articles/fair-model-comparisons.md", "full_draft_read": true, "review_body_words": 1134, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Fixed versus optimised designs and prompt-format qualification are sound; advice is mostly uncited in this draft.", "claim_ids": [ "A-fair-comparison-01", "A-fair-comparison-02", "A-fair-comparison-03" ] }, "practical_usefulness": { "score": 4, "evidence": "Freezing prompts/tools, equal tuning budgets, retry handling and configuration ledger are actionable.", "article_file": "A/articles/fair-model-comparisons.md" }, "writing_quality": { "score": 4, "evidence": "Good distinction between deployment choice and component attribution; manifest remains a prose list.", "article_file": "A/articles/fair-model-comparisons.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: How to compare AI models fairly. Query: fair AI model comparison. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Supply a filled configuration ledger and a prompt-format primary citation.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "latency-cost", "article_file": "A/articles/latency-cost.md", "full_draft_read": true, "review_body_words": 1093, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Pricing categories and example totals verified; initial-versus-retry charge separation needs an explicit definition.", "claim_ids": [ "A-latency-cost-01", "A-latency-cost-02", "A-latency-cost-03", "A-latency-cost-04" ] }, "practical_usefulness": { "score": 4, "evidence": "User-facing clock boundaries and human-review arithmetic expose economics beyond token rates.", "article_file": "A/articles/latency-cost.md" }, "writing_quality": { "score": 4, "evidence": "Readable cost worksheet narrative; final per-success denominator is left to the reader’s chosen definition.", "article_file": "A/articles/latency-cost.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Measure AI latency and cost per successful task. Query: AI model latency and cost per task. Description matches the draft’s reader question; no search volumes asserted. Uppercase slug segments need routine URL-style normalisation. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Use all-attempt totals once; finish the worksheet with accepted-task counts and ratios.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "agent-evaluation", "article_file": "A/articles/agent-system-evaluation.md", "full_draft_read": true, "review_body_words": 1328, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Trace guide and 2026 SWE-Bench Pro Verified study support the stated narrow claims.", "claim_ids": [ "A-agent-evaluation-01", "A-agent-evaluation-02", "A-agent-evaluation-03", "A-agent-evaluation-04" ] }, "practical_usefulness": { "score": 4, "evidence": "External acceptance conditions, resettable coding task, side effects and ablations address real system boundaries.", "article_file": "A/articles/agent-system-evaluation.md" }, "writing_quality": { "score": 4, "evidence": "Strong model/system distinction and failure controls; the final multi-link footer is crowded.", "article_file": "A/articles/agent-system-evaluation.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Evaluate the agent system, not only its language model. Query: AI agent evaluation and tool use. Description matches the draft’s reader question; no search volumes asserted. Uppercase slug segments need routine URL-style normalisation. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Add a concrete lost-write-acknowledgement recovery case and prioritise the next reading step.", "publication_blocker_claim_ids": [] }, { "cluster": "A", "article_id": "evaluation-maintenance", "article_file": "A/articles/maintaining-evaluations.md", "full_draft_read": true, "review_body_words": 1064, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Continuous-evaluation reference and grader-revalidation advice verified; illustrative cadence not passed off as measured.", "claim_ids": [ "A-evaluation-maintenance-01", "A-evaluation-maintenance-02", "A-evaluation-maintenance-03" ] }, "practical_usefulness": { "score": 4, "evidence": "Stable anchor plus rotating set, bridge runs, versioning and ownership form a usable maintenance routine.", "article_file": "A/articles/maintaining-evaluations.md" }, "writing_quality": { "score": 4, "evidence": "Clear operational progression and privacy caveats; lacks a completed change/decision example.", "article_file": "A/articles/maintaining-evaluations.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Keep AI evaluations useful as models and products change. Query: maintain AI model evaluations. Description matches the draft’s reader question; no search volumes asserted. Uppercase slug segments need routine URL-style normalisation. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "A/cluster-map.json" } }, "principal_edit": "Add a version-change record showing task drift versus model regression.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "P01", "article_file": "B/articles/ai-model-benchmarking-guide.md", "full_draft_read": true, "review_body_words": 1938, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "MMLU and official evaluation guidance faithfully used; invented $12/80 arithmetic correct and labelled.", "claim_ids": [ "B-P01-01", "B-P01-02", "B-P01-03", "B-P01-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Six deliverable artefacts, decision contract and bounded support-assistant journey create a practical first evaluation.", "article_file": "B/articles/ai-model-benchmarking-guide.md" }, "writing_quality": { "score": 4, "evidence": "Readable, precise and consistently conditional; closing first-evaluation instructions repeat earlier sections.", "article_file": "B/articles/ai-model-benchmarking-guide.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: AI model benchmarking: a practical guide to choosing with evidence. Query: AI model benchmarking. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "A compact worked decision memo would further reduce implementation effort.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S01", "article_file": "B/articles/read-ai-benchmark-leaderboards.md", "full_draft_read": true, "review_body_words": 1152, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "HumanEval scope/attempt definition, HELM and dated SWE-bench common-harness view verified.", "claim_ids": [ "B-S01-01", "B-S01-02", "B-S01-03", "B-S01-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Evidence-card fields connect public evidence to explicit local follow-up tests.", "article_file": "B/articles/read-ai-benchmark-leaderboards.md" }, "writing_quality": { "score": 4, "evidence": "Clear question-led audit; some warning paragraphs overlap the pillar.", "article_file": "B/articles/read-ai-benchmark-leaderboards.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: How to read an AI benchmark leaderboard without overreading it. Query: how to read AI benchmarks. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Include one filled evidence card; refresh the view-specific maintainer claim before publication.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S02", "article_file": "B/articles/design-llm-evaluation-tasks.md", "full_draft_read": true, "review_body_words": 1084, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "CheckList claim and related-case/timing guidance match checked primary/official sources.", "claim_ids": [ "B-S02-01", "B-S02-02", "B-S02-03" ] }, "practical_usefulness": { "score": 4, "evidence": "RETURN-014 missing-date case, case schema and contrastive/invariant tests turn requirements into behaviour.", "article_file": "B/articles/design-llm-evaluation-tasks.md" }, "writing_quality": { "score": 4, "evidence": "Specific observation-point explanation is strong; repeated procedural labels can be reduced.", "article_file": "B/articles/design-llm-evaluation-tasks.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Design an LLM evaluation around tasks your users actually do. Query: LLM evaluation task design. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Render the proposed case schema as a filled record.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S03", "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md", "full_draft_read": true, "review_body_words": 1103, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Four relevant primary/official records support leakage, detection limits, datasheets and exchangeability.", "claim_ids": [ "B-S03-01", "B-S03-02", "B-S03-03", "B-S03-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Provenance, grouped splitting, context assembly and exposed-holdout transition are concretely explained.", "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md" }, "writing_quality": { "score": 4, "evidence": "Clean distinction between contamination, leakage and mismatch; sections retain a consistent reader problem.", "article_file": "B/articles/evaluation-dataset-leakage-holdouts.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Build an evaluation dataset that resists leakage and contamination. Query: LLM evaluation dataset leakage. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Add a brief example overlap report; no substantive factual repair found in checked claims.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S04", "article_file": "B/articles/llm-evaluation-metrics-rubrics.md", "full_draft_read": true, "review_body_words": 1065, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Confusion-matrix arithmetic, precision/recall/F1 and averaging verified; gates and examples bounded.", "claim_ids": [ "B-S04-01", "B-S04-02", "B-S04-03" ] }, "practical_usefulness": { "score": 5, "evidence": "Worked 100-request matrix plus 0/1/2 policy rubric and scorer-negative cases provide followable scoring.", "article_file": "B/articles/llm-evaluation-metrics-rubrics.md" }, "writing_quality": { "score": 4, "evidence": "Precise denominator handling and syntax/semantics distinction; text is dense in places but clear.", "article_file": "B/articles/llm-evaluation-metrics-rubrics.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Choose LLM evaluation metrics and write a rubric that works. Query: LLM evaluation metrics. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Light copy edit; optionally put the matrix and gate rules in one reusable table.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S05", "article_file": "B/articles/human-reviewers-llm-judges.md", "full_draft_read": true, "review_body_words": 1117, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Judge-study scope accurately attributed without transferring its agreement rate; always-pass example correct.", "claim_ids": [ "B-S05-01", "B-S05-02", "B-S05-03", "B-S05-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Separates calibration and validation; audits false passes, grader injection, order flips and unscorable evidence.", "article_file": "B/articles/human-reviewers-llm-judges.md" }, "writing_quality": { "score": 4, "evidence": "Well-controlled claims and concrete failure explanations; repeated injunctions make the rhythm uniform.", "article_file": "B/articles/human-reviewers-llm-judges.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Human reviewers and LLM judges: build a grading process you can trust. Query: LLM as a judge evaluation. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Add a filled confusion matrix or adjudication example for faster adoption.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S06", "article_file": "B/articles/benchmark-uncertainty-repeatability.md", "full_draft_read": true, "review_body_words": 1055, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Wilson interval recalculated; frequentist coverage and paired-bootstrap definitions checked.", "claim_ids": [ "B-S06-01", "B-S06-02", "B-S06-03", "B-S06-04", "B-S06-05" ] }, "practical_usefulness": { "score": 5, "evidence": "Paired outcome counts, interval assumptions, resampling recipe and non-inferiority/stopping cautions supply real technical depth.", "article_file": "B/articles/benchmark-uncertainty-repeatability.md" }, "writing_quality": { "score": 4, "evidence": "Accessible statistics with caveats for sparse/degenerate samples; title/query promise broad significance advice in limited space.", "article_file": "B/articles/benchmark-uncertainty-repeatability.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Is that benchmark improvement real? Repeats and uncertainty. Query: LLM benchmark statistical significance. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Light technical copy edit; keep the independent-unit and interval-method qualifications.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S07", "article_file": "B/articles/fair-ai-model-comparisons.md", "full_draft_read": true, "review_body_words": 1087, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Sclar finding, reasoning billing and harness description verified with correct scope.", "claim_ids": [ "B-S07-01", "B-S07-02", "B-S07-03" ] }, "practical_usefulness": { "score": 4, "evidence": "Unsupported settings, prior tuning investment, native tools and cost envelopes are handled explicitly.", "article_file": "B/articles/fair-ai-model-comparisons.md" }, "writing_quality": { "score": 4, "evidence": "Clear and bounded; run manifest fields are spread across prose paragraphs.", "article_file": "B/articles/fair-ai-model-comparisons.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Compare AI models fairly across prompts, tools and settings. Query: fair AI model comparison. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Provide a compact filled manifest; refresh provider-specific documentation.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S08", "article_file": "B/articles/ai-latency-cost-per-successful-task.md", "full_draft_read": true, "review_body_words": 1094, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "All pricing-category/cache/billing assertions and both invented arithmetic examples checked.", "claim_ids": [ "B-S08-01", "B-S08-02", "B-S08-03", "B-S08-04", "B-S08-05", "B-S08-06" ] }, "practical_usefulness": { "score": 5, "evidence": "Disjoint usage accounting, undefined zero-success ratio, censored latency, warm/cold cache and repair-cost scenarios are unusually usable.", "article_file": "B/articles/ai-latency-cost-per-successful-task.md" }, "writing_quality": { "score": 4, "evidence": "Precise economic explanation; proposed target query is a slightly awkward keyword bundle.", "article_file": "B/articles/ai-latency-cost-per-successful-task.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Benchmark AI latency and cost per successful task. Query: AI model latency cost benchmark. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Polish query/description and provide a worksheet table; preserve the hypothetical labels.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S09", "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md", "full_draft_read": true, "review_body_words": 1092, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "τ-bench state and pass^k definitions verified; averaging probabilities versus powering a mean correctly distinguished.", "claim_ids": [ "B-S09-01", "B-S09-02", "B-S09-03", "B-S09-04" ] }, "practical_usefulness": { "score": 5, "evidence": "Permission predicates, lost acknowledgements, idempotent recovery, simulator limits and reset rules make the system test concrete.", "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md" }, "writing_quality": { "score": 4, "evidence": "Detailed yet readable, with necessary distinctions between outcome, permission and reporting failures.", "article_file": "B/articles/evaluate-ai-agents-tools-outcomes.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Evaluate AI agents by outcomes, tool actions and recovery. Query: AI agent evaluation. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Add a short scenario record for copying; no checked publication blocker.", "publication_blocker_claim_ids": [] }, { "cluster": "B", "article_id": "S10", "article_file": "B/articles/maintain-llm-evaluation-suite.md", "full_draft_read": true, "review_body_words": 1085, "scores": { "accuracy_and_sources": { "score": 5, "evidence": "Continuous-evaluation statement supported; incident sampling and scorer-drift reasoning appropriately bounded.", "claim_ids": [ "B-S10-01", "B-S10-02", "B-S10-03" ] }, "practical_usefulness": { "score": 4, "evidence": "Regression triage, ownership, immutable labels and decision-expiry triggers make maintenance actionable.", "article_file": "B/articles/maintain-llm-evaluation-suite.md" }, "writing_quality": { "score": 4, "evidence": "Strong policy-change example and clear separation of diagnostic/representative evidence; some repeated versioning reminders.", "article_file": "B/articles/maintain-llm-evaluation-suite.md" }, "intent_and_metadata": { "score": 5, "evidence": "Title: Keep an LLM evaluation suite useful after the first model choice. Query: continuous LLM evaluation. Description matches the draft’s reader question; no search volumes asserted. Specific, natural reader question and metadata promise; only final publication checks remain.", "metadata_source": "B/cluster-map.json" } }, "principal_edit": "Condense repeated reminders and provide a filled release record.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-00", "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md", "full_draft_read": true, "review_body_words": 2037, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Numerous historical counts/results and power arithmetic verified; architecture sentence overstates what workflows add.", "claim_ids": [ "C-ai-eval-00-01", "C-ai-eval-00-02", "C-ai-eval-00-03", "C-ai-eval-00-04", "C-ai-eval-00-05", "C-ai-eval-00-06", "C-ai-eval-00-07", "C-ai-eval-00-08", "C-ai-eval-00-09", "C-ai-eval-00-10", "C-ai-eval-00-11", "C-ai-eval-00-12", "C-ai-eval-00-13", "C-ai-eval-00-14" ] }, "practical_usefulness": { "score": 4, "evidence": "Six-step loop, decision memo and worked uncertainty example give useful orientation; numerical minimums require more conditional framing.", "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md" }, "writing_quality": { "score": 4, "evidence": "Engaging opening and clear headings; dense catalogue of studies and categorical demo/evaluation rhetoric distract from the first job.", "article_file": "C/articles/evaluating-ai-models-and-agent-systems.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Evaluating AI Models and Agent Systems: A Practical Guide for Technical Teams. Query: how to evaluate AI models for production. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Correct workflow attribution, label the opening scenario illustrative and bound small-sample advice.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-01", "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md", "full_draft_read": true, "review_body_words": 1370, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "MMLU/BIG-bench/HELM/Arena/BetterBench/Leaderboard Illusion numbers supported with historical dating.", "claim_ids": [ "C-ai-eval-01-01", "C-ai-eval-01-02", "C-ai-eval-01-03", "C-ai-eval-01-04", "C-ai-eval-01-05", "C-ai-eval-01-06" ] }, "practical_usefulness": { "score": 4, "evidence": "Shortlist scorecard and seven-question audit are usable; generic pilot ranges do not guarantee a selection decision.", "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md" }, "writing_quality": { "score": 4, "evidence": "Readable construct-validity explanation; substantial benchmark-history overlap with pillar and agent article.", "article_file": "C/articles/public-benchmarks-vs-workload-evaluation.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Public Benchmarks Are a Screening Tool, Not a Decision. Query: public benchmarks vs custom AI evals. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Tighten the history and qualify selection/replication absolutes.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-02", "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md", "full_draft_read": true, "review_body_words": 1373, "scores": { "accuracy_and_sources": { "score": 3, "evidence": "Criteria-drift source checked; sample-size and same-generator bans go beyond evidence despite rough-guide caveats.", "claim_ids": [ "C-ai-eval-02-01", "C-ai-eval-02-02", "C-ai-eval-02-03", "C-ai-eval-02-04" ] }, "practical_usefulness": { "score": 4, "evidence": "Copyable task schema, stratification and labelling protocol have depth; grouping, privacy and independent label validation need stronger treatment.", "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md" }, "writing_quality": { "score": 3, "evidence": "Concrete structure but frequent categorical claims about demos, synthetic data and multi-skill tasks overstate the case.", "article_file": "C/articles/designing-evaluation-tasks-and-datasets.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Designing Evaluation Tasks and Datasets That Reflect Your Workload. Query: how to design an AI evaluation dataset. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Make size rules conditional; keep end-to-end tasks plus component diagnostics; add provenance/access safeguards.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-03", "article_file": "C/articles/contamination-leakage-and-holdouts.md", "full_draft_read": true, "review_body_words": 1391, "scores": { "accuracy_and_sources": { "score": 3, "evidence": "Contamination figures corroborated; LiveCodeBench initial-release label wrong; provider opacity establishes no performance upper bound; survey is secondary.", "claim_ids": [ "C-ai-eval-03-01", "C-ai-eval-03-02", "C-ai-eval-03-03", "C-ai-eval-03-04", "C-ai-eval-03-05" ] }, "practical_usefulness": { "score": 3, "evidence": "Layered controls and custody template useful; cutoff assumptions and one/two-decision retirement policy need repair.", "article_file": "C/articles/contamination-leakage-and-holdouts.md" }, "writing_quality": { "score": 4, "evidence": "Distinct mechanisms explained clearly, but confident threat-control wording exceeds what the checks establish.", "article_file": "C/articles/contamination-leakage-and-holdouts.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Contamination, Leakage and Holdouts: Why Your Benchmark Score May Be Inflated. Query: AI benchmark contamination and data leakage. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Replace the unsupported upper-bound rule; identify version/cutoff uncertainty and reconcile retirement rules.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-04", "article_file": "C/articles/objective-metrics-and-rubrics.md", "full_draft_read": true, "review_body_words": 1366, "scores": { "accuracy_and_sources": { "score": 3, "evidence": "pass@k formula and historical benchmark claims verified; F1 verbosity and traffic/micro generalisations need qualification.", "claim_ids": [ "C-ai-eval-04-01", "C-ai-eval-04-02", "C-ai-eval-04-03", "C-ai-eval-04-04", "C-ai-eval-04-05" ] }, "practical_usefulness": { "score": 4, "evidence": "Metric families, rubric table, gates and scorer audit are practical assets.", "article_file": "C/articles/objective-metrics-and-rubrics.md" }, "writing_quality": { "score": 4, "evidence": "Clear metric-to-decision structure; slogan-like capability/dependability split is too absolute.", "article_file": "C/articles/objective-metrics-and-rubrics.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Objective Metrics and Rubrics That Measure What Matters. Query: AI evaluation metrics and rubrics. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Specify n≥k/task averaging; explain normalisation boundaries and metric-specific length effects.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-05", "article_file": "C/articles/human-evaluation-and-model-judges.md", "full_draft_read": true, "review_body_words": 1300, "scores": { "accuracy_and_sources": { "score": 3, "evidence": "Historical agreement and criteria drift supported; self-enhancement certainty exceeds cited study and agreement methods lack nearby sourcing.", "claim_ids": [ "C-ai-eval-05-01", "C-ai-eval-05-02", "C-ai-eval-05-03", "C-ai-eval-05-04", "C-ai-eval-05-05" ] }, "practical_usefulness": { "score": 3, "evidence": "Validation protocol and agreement formula useful; order averaging needs vote encoding/reversal reporting and mitigation limits.", "article_file": "C/articles/human-evaluation-and-model-judges.md" }, "writing_quality": { "score": 3, "evidence": "Accessible taxonomy, but Mitigations that work and universal comparative claims sound more conclusive than the evidence.", "article_file": "C/articles/human-evaluation-and-model-judges.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Human Evaluation and Model Judges: Getting Reliable Judgements at Scale. Query: LLM as a judge reliability. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Define order aggregation and disagreement handling; qualify bias/mitigation claims and source agreement statistics.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-06", "article_file": "C/articles/repeatability-variance-and-uncertainty.md", "full_draft_read": true, "review_body_words": 1426, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "SE/power/repeat arithmetic and cited historical results verified; normal-interval and effective-sample-size boundaries need clearer limits.", "claim_ids": [ "C-ai-eval-06-01", "C-ai-eval-06-02", "C-ai-eval-06-03", "C-ai-eval-06-04", "C-ai-eval-06-05", "C-ai-eval-06-06", "C-ai-eval-06-07", "C-ai-eval-06-08", "C-ai-eval-06-09" ] }, "practical_usefulness": { "score": 4, "evidence": "Formulas, paired example, clustering, power and reporting table add substantial technical depth.", "article_file": "C/articles/repeatability-variance-and-uncertainty.md" }, "writing_quality": { "score": 4, "evidence": "Well sequenced for technical readers; shorthand claims about every-time reliability and fixed sample counts overreach.", "article_file": "C/articles/repeatability-variance-and-uncertainty.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Repeatability, Variance and Uncertainty in Model Evaluations. Query: AI eval statistical significance error bars. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "State normal-approximation limits, conditional clustering effect and the task-level pass^k estimator.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-07", "article_file": "C/articles/prompt-tool-and-configuration-fairness.md", "full_draft_read": true, "review_body_words": 1269, "scores": { "accuracy_and_sources": { "score": 3, "evidence": "Maintainer/Inspect claims verified; effort always increasing work is not guaranteed and same nominal labels do not match compute.", "claim_ids": [ "C-ai-eval-07-01", "C-ai-eval-07-02", "C-ai-eval-07-03", "C-ai-eval-07-04", "C-ai-eval-07-05" ] }, "practical_usefulness": { "score": 3, "evidence": "Copyable manifest and confounder audit useful; effort and unsupported-setting treatment could mislead a cross-provider comparison.", "article_file": "C/articles/prompt-tool-and-configuration-fairness.md" }, "writing_quality": { "score": 4, "evidence": "Clear experiment framing and concrete hypothetical confounders; templates look more universally portable than they are.", "article_file": "C/articles/prompt-tool-and-configuration-fairness.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Comparing Models Fairly: Prompts, Tools and Configuration. Query: fair comparison of AI models prompts tools. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Label model-specific/unsupported controls and compare measured envelopes rather than nominal effort equivalence.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-08", "article_file": "C/articles/latency-token-usage-and-cost.md", "full_draft_read": true, "review_body_words": 1332, "scores": { "accuracy_and_sources": { "score": 2, "evidence": "Most live rates, promotion and arithmetic corroborated. Haiku long-context bracket omitted; visible-output definition conflicts with reasoning-inclusive billing.", "claim_ids": [ "C-ai-eval-08-01", "C-ai-eval-08-02", "C-ai-eval-08-03", "C-ai-eval-08-04", "C-ai-eval-08-05", "C-ai-eval-08-06", "C-ai-eval-08-07", "C-ai-eval-08-08", "C-ai-eval-08-09", "C-ai-eval-08-10", "C-ai-eval-08-11" ] }, "practical_usefulness": { "score": 3, "evidence": "Cost-quality table and marginal-success calculation helpful, but accounting ambiguity can materially misprice a real run.", "article_file": "C/articles/latency-token-usage-and-cost.md" }, "writing_quality": { "score": 4, "evidence": "Clear numerical walk-through; cost symbols lack explicit units/disjoint categories and warm-cache recipe is underspecified.", "article_file": "C/articles/latency-token-usage-and-cost.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Latency, Token Usage and Cost: The Other Half of the Scorecard. Query: AI cost per task and latency benchmarking. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Repair billed-output/retry accounting and all price conditions before publication; add cache-write assumptions.", "publication_blocker_claim_ids": [ "C-ai-eval-08-03" ] }, { "cluster": "C", "article_id": "ai-eval-09", "article_file": "C/articles/evaluating-agents-and-tool-use.md", "full_draft_read": true, "review_body_words": 1383, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "Original benchmark counts/results, maintainer subsets, trace guidance and Inspect support verified.", "claim_ids": [ "C-ai-eval-09-01", "C-ai-eval-09-02", "C-ai-eval-09-03", "C-ai-eval-09-04", "C-ai-eval-09-05", "C-ai-eval-09-06", "C-ai-eval-09-07", "C-ai-eval-09-08" ] }, "practical_usefulness": { "score": 4, "evidence": "System decomposition, resettable fixtures, ablations and failure taxonomy are useful; recovery tests need concrete examples.", "article_file": "C/articles/evaluating-agents-and-tool-use.md" }, "writing_quality": { "score": 4, "evidence": "Strong boundary explanation; repeats much benchmark history and presents pass^k as universally predictive.", "article_file": "C/articles/evaluating-agents-and-tool-use.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Evaluating Agents and Tool Use: Model Benchmarks Are Not System Benchmarks. Query: how to evaluate AI agents and tool use. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Bound reliability inference and add an idempotency/recovery scenario; shorten repeated benchmark survey.", "publication_blocker_claim_ids": [] }, { "cluster": "C", "article_id": "ai-eval-10", "article_file": "C/articles/interpreting-results-and-maintaining-evals.md", "full_draft_read": true, "review_body_words": 1322, "scores": { "accuracy_and_sources": { "score": 4, "evidence": "BetterBench/selection-effect claims and scheduled Evals retirement corroborated; saturation and holdout rules need qualification.", "claim_ids": [ "C-ai-eval-10-01", "C-ai-eval-10-02", "C-ai-eval-10-03", "C-ai-eval-10-04", "C-ai-eval-10-05" ] }, "practical_usefulness": { "score": 4, "evidence": "Decision memo and ownership/cadence model actionable; gate-preserving retirement and grader-change bridging need more detail.", "article_file": "C/articles/interpreting-results-and-maintaining-evals.md" }, "writing_quality": { "score": 4, "evidence": "Clear closing operational article; maturity hierarchy and strong pilot labels are more normative than necessary.", "article_file": "C/articles/interpreting-results-and-maintaining-evals.md" }, "intent_and_metadata": { "score": 4, "evidence": "Title: Interpreting Results and Keeping Evaluations Useful Over Time. Query: how to interpret AI benchmark results. Description matches the draft’s reader question; no search volumes asserted. Useful intent match; minor title/query or description refinement remains.", "metadata_source": "C/cluster-map.json" } }, "principal_edit": "Keep critical saturated regression checks, reconcile holdout policy and refresh the dated notice.", "publication_blocker_claim_ids": [] } ], "article_count": 33 }