{ "scale": "Frozen 0–5 anchors; decimals interpolate editorial judgement, not statistical confidence. Article scores assigned independently before pairwise passes.", "clusters": { "A": { "articles": [ { "id": "P01", "file": "articles/ai-model-benchmarking-guide.md", "scores": { "accuracy_and_sources": 4.6, "practical_usefulness": 4.5, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/ai-model-benchmarking-guide.md:67 (Score what success means)", "finding": "Sourced task-specific evaluation is accurate; MMLU scope and cost arithmetic also check out. Clear fact/recommendation/example boundaries.", "check_ids": [ "A-P01-01", "A-P01-02", "A-P01-03" ] }, { "dimension": "practical_usefulness", "location": "A/articles/ai-model-benchmarking-guide.md:19 (Start with a decision contract)", "finding": "Gates, six deliverable artefacts and a bounded first run provide an executable management plan; one filled decision-record example would make the handoff easier.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/ai-model-benchmarking-guide.md:109 (Make a decision that can be reviewed later)", "finding": "Clear end-to-end sequence and concrete review triggers; many short imperative paragraphs make the closing sections somewhat checklist-like.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/ai-model-benchmarking-guide.md:1 (front matter)", "finding": "Title, description and AI model benchmarking query match the broad decision guide; map gives a specific reader question and all fields agree.", "check_ids": [] } ] }, { "id": "S01", "file": "articles/read-ai-benchmark-leaderboards.md", "scores": { "accuracy_and_sources": 4.6, "practical_usefulness": 4.4, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/read-ai-benchmark-leaderboards.md:39 (Check whether the rows are comparable)", "finding": "Dated SWE-bench view matches the maintainer; HumanEval and HELM descriptions are faithful and avoid transferring historical scores.", "check_ids": [ "A-S01-01", "A-S01-02", "A-S01-03", "A-S01-04" ] }, { "dimension": "practical_usefulness", "location": "A/articles/read-ai-benchmark-leaderboards.md:77 (Turn the reading into a shortlist)", "finding": "Evidence-card fields and a local follow-up test connect public scores to action; no completed card demonstrates filling unknowns.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/read-ai-benchmark-leaderboards.md:29 (Inspect what earns the score)", "finding": "Explains attempt budgets and unlike metrics in ordinary language; the audit sequence is easy to follow, although several cautions recur later.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/read-ai-benchmark-leaderboards.md:1 (front matter)", "finding": "Reader intent is interpreting leaderboards; title and description accurately promise protocol interpretation, with a plausible query and matching map.", "check_ids": [] } ] }, { "id": "S02", "file": "articles/design-llm-evaluation-tasks.md", "scores": { "accuracy_and_sources": 4.5, "practical_usefulness": 4.3, "writing_quality": 4.3, "intent_and_metadata": 4.6 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/design-llm-evaluation-tasks.md:49 (Build a coverage matrix)", "finding": "CheckList precedent is supported; related-case grouping is sound. Most local workflow design is appropriately presented as recommendation.", "check_ids": [ "A-S02-01", "A-S02-02" ] }, { "dimension": "practical_usefulness", "location": "A/articles/design-llm-evaluation-tasks.md:39 (Give every case a usable schema)", "finding": "RETURN-014 distinguishes model input, reference rationale and acceptable clarification. A populated schema and actual coverage table would reduce translation work.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/design-llm-evaluation-tasks.md:29 (Choose the unit of work)", "finding": "Observation-point example makes task boundaries precise; repeated contract/schema/coverage instructions could be compressed.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/design-llm-evaluation-tasks.md:1 (front matter)", "finding": "Specific task-design query and behavioural promise fit the missing-date example; all metadata fields align with the map.", "check_ids": [] } ] }, { "id": "S03", "file": "articles/evaluation-dataset-leakage-holdouts.md", "scores": { "accuracy_and_sources": 4.7, "practical_usefulness": 4.5, "writing_quality": 4.4, "intent_and_metadata": 4.6 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/evaluation-dataset-leakage-holdouts.md:57 (Audit overlap at several levels)", "finding": "Canonical-versus-shuffled detection retains its assumption and access limits; leakage, detection and dataset documentation citations all support their claims.", "check_ids": [ "A-S03-01", "A-S03-02", "A-S03-03", "A-S03-04" ] }, { "dimension": "practical_usefulness", "location": "A/articles/evaluation-dataset-leakage-holdouts.md:47 (Protect the holdout from repeated decisions)", "finding": "Explains exposed-set promotion, grouped splits, context-index overlap and label corrections. A worked overlap disposition would improve implementation detail.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/evaluation-dataset-leakage-holdouts.md:17 (Distinguish three routes to misleading evidence)", "finding": "Leakage, contamination and mismatch are distinguished cleanly; disciplined caveats preserve readability despite a dense subject.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/evaluation-dataset-leakage-holdouts.md:1 (front matter)", "finding": "Dataset-leakage query and description fit both operational controls and limits of contamination detection; no unsupported clean-data promise.", "check_ids": [] } ] }, { "id": "S04", "file": "articles/llm-evaluation-metrics-rubrics.md", "scores": { "accuracy_and_sources": 4.7, "practical_usefulness": 4.6, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/llm-evaluation-metrics-rubrics.md:37 (Work through a classification example)", "finding": "Confusion-matrix counts, metric definitions and averaging are correct; invented inputs are clearly labelled.", "check_ids": [ "A-S04-01", "A-S04-02", "A-S04-03" ] }, { "dimension": "practical_usefulness", "location": "A/articles/llm-evaluation-metrics-rubrics.md:49 (Write an anchored rubric for open answers)", "finding": "0/1/2 policy anchors, denominators, critical flags and scorer adversaries are directly usable. One scored answer would further clarify boundaries.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/llm-evaluation-metrics-rubrics.md:59 (Separate release gates from preferences)", "finding": "Keeps gates distinct from style and composites with concise examples; prose is clear, though a visual confusion matrix would aid scanning.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/llm-evaluation-metrics-rubrics.md:1 (front matter)", "finding": "Metrics query, title and description reflect the actual arithmetic and rubric content; metadata agrees throughout.", "check_ids": [] } ] }, { "id": "S05", "file": "articles/human-reviewers-llm-judges.md", "scores": { "accuracy_and_sources": 4.6, "practical_usefulness": 4.5, "writing_quality": 4.4, "intent_and_metadata": 4.6 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/human-reviewers-llm-judges.md:47 (Check known bias mechanisms)", "finding": "Original judge paper supports named bias mechanisms; no borrowed agreement percentage. The 90%-agreement counterexample is correct.", "check_ids": [ "A-S05-01", "A-S05-02" ] }, { "dimension": "practical_usefulness", "location": "A/articles/human-reviewers-llm-judges.md:59 (Measure errors that affect the decision)", "finding": "Audits false passes, class composition and ordinary passes as well as disagreements; could add a filled judge-output schema and one adjudication example.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/human-reviewers-llm-judges.md:71 (Route uncertainty instead of hiding it)", "finding": "Separates evaluator uncertainty from correct candidate clarification, a useful precision; calibration/versioning prose is orderly but familiar across the series.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/human-reviewers-llm-judges.md:1 (front matter)", "finding": "Title/query target judge validation and description names concrete controls that the article actually provides; all fields match.", "check_ids": [] } ] }, { "id": "S06", "file": "articles/benchmark-uncertainty-repeatability.md", "scores": { "accuracy_and_sources": 4.7, "practical_usefulness": 4.5, "writing_quality": 4.2, "intent_and_metadata": 4.6 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/benchmark-uncertainty-repeatability.md:37 (Put a single success rate in context)", "finding": "Wilson result and coverage interpretation verify; paired totals and bootstrap procedure are correct with assumptions and failure modes retained.", "check_ids": [ "A-S06-01", "A-S06-02", "A-S06-03", "A-S06-04" ] }, { "dimension": "practical_usefulness", "location": "A/articles/benchmark-uncertainty-repeatability.md:47 (Estimate the difference with paired resampling)", "finding": "Gives 10,000-resample recipe, paired indices, group weighting and small-sample limits. A short worked resample or pseudocode would bridge prose to implementation.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/benchmark-uncertainty-repeatability.md:57 (Repeat the protocol without inflating the sample)", "finding": "Precise nesting and non-inferiority cautions; technical terms accumulate rapidly and a small glossary would help the founder audience.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/benchmark-uncertainty-repeatability.md:1 (front matter)", "finding": "Significance-oriented query is answered with estimates and decision margins rather than a false yes/no test; title and description fit.", "check_ids": [] } ] }, { "id": "S07", "file": "articles/fair-ai-model-comparisons.md", "scores": { "accuracy_and_sources": 4.7, "practical_usefulness": 4.5, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/fair-ai-model-comparisons.md:47 (Match capabilities without inventing equivalence)", "finding": "Prompt-sensitivity, reasoning billing and harness descriptions verify; provider-specific conditions are properly bounded.", "check_ids": [ "A-S07-01", "A-S07-02", "A-S07-03" ] }, { "dimension": "practical_usefulness", "location": "A/articles/fair-ai-model-comparisons.md:67 (Save a run manifest someone else can use)", "finding": "Manifest covers request structure, local inference settings, cache, tuning budget and retained content. A filled manifest would improve reuse.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/fair-ai-model-comparisons.md:17 (State which comparison you mean)", "finding": "Controlled and deployment-optimised experiments stay distinct throughout; strong concrete support-assistant framing with some repeated warnings.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/fair-ai-model-comparisons.md:1 (front matter)", "finding": "Fair-comparison title/query and metadata correctly promise prompts/tools/settings controls and separate tuned systems.", "check_ids": [] } ] }, { "id": "S08", "file": "articles/ai-latency-cost-per-successful-task.md", "scores": { "accuracy_and_sources": 4.6, "practical_usefulness": 4.6, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/ai-latency-cost-per-successful-task.md:47 (Make cache conditions visible)", "finding": "Current billing/cache/streaming attributions verify; no supplier rates are passed off as measured. All three cost calculations are correct.", "check_ids": [ "A-S08-01", "A-S08-02", "A-S08-03", "A-S08-04", "A-S08-05" ] }, { "dimension": "practical_usefulness", "location": "A/articles/ai-latency-cost-per-successful-task.md:57 (Calculate cost per accepted task)", "finding": "Shows usage-to-charge and spend-to-success arithmetic, zero-success handling, timeouts and cache scenarios. A usage-row worksheet would complete the accounting example.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/ai-latency-cost-per-successful-task.md:17 (Define the task clock)", "finding": "Clearly separates first content, useful completion and deadline success; ratio and dominated-choice sections maintain a concrete operational focus.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/ai-latency-cost-per-successful-task.md:1 (front matter)", "finding": "Cost-per-success title and latency/cost query align with the worked calculation and measured-boundary advice; metadata matches the map.", "check_ids": [] } ] }, { "id": "S09", "file": "articles/evaluate-ai-agents-tools-outcomes.md", "scores": { "accuracy_and_sources": 4.7, "practical_usefulness": 4.6, "writing_quality": 4.4, "intent_and_metadata": 4.7 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/evaluate-ai-agents-tools-outcomes.md:69 (Measure consistency with the right metric)", "finding": "τ-bench state evaluation and pass^k/pass@k distinction are correct, including aggregation caution. State success is not conflated with permission compliance.", "check_ids": [ "A-S09-01", "A-S09-02", "A-S09-03" ] }, { "dimension": "practical_usefulness", "location": "A/articles/evaluate-ai-agents-tools-outcomes.md:59 (Test recovery deliberately)", "finding": "Lost write acknowledgement, reconciliation, reset state and separate trajectory checks are substantial implementation detail; an explicit predicate or tool-contract specimen would help.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/evaluate-ai-agents-tools-outcomes.md:49 (Inspect the path to diagnose failures)", "finding": "Organises selection, arguments, policy and reporting failures coherently; examples remain concrete without implying a live deployment.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/evaluate-ai-agents-tools-outcomes.md:1 (front matter)", "finding": "Outcome/tool/recovery title and description accurately identify the unique agent-system question; query and map agree.", "check_ids": [] } ] }, { "id": "S10", "file": "articles/maintain-llm-evaluation-suite.md", "scores": { "accuracy_and_sources": 4.5, "practical_usefulness": 4.4, "writing_quality": 4.3, "intent_and_metadata": 4.6 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "A/articles/maintain-llm-evaluation-suite.md:17 (Give the evidence an owner)", "finding": "Continuous-evaluation attribution verifies; exposed regression versus protected holdout distinction is sound and no universal cadence is asserted.", "check_ids": [ "A-S10-01", "A-S10-02" ] }, { "dimension": "practical_usefulness", "location": "A/articles/maintain-llm-evaluation-suite.md:47 (Triage a regression before changing the model)", "finding": "Provides input/output/score triage, dispositions and expiry triggers. A completed change record would make responsibilities and evidence easier to implement.", "check_ids": [] }, { "dimension": "writing_quality", "location": "A/articles/maintain-llm-evaluation-suite.md:57 (Turn incidents into useful cases)", "finding": "Date-convention incident gives a concrete mechanism; layered suites and review ownership are clear, with some repetition of data/judge guidance.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "A/articles/maintain-llm-evaluation-suite.md:1 (front matter)", "finding": "Continuous-evaluation query and maintenance promise match versioning, ownership and incident handling; fields agree with map.", "check_ids": [] } ] } ] }, "B": { "articles": [ { "id": "model-benchmarking", "file": "articles/choosing-model-benchmark.md", "scores": { "accuracy_and_sources": 4.4, "practical_usefulness": 4.3, "writing_quality": 4.4, "intent_and_metadata": 4.5 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/choosing-model-benchmark.md:31 (Use public benchmarks as a map, not a verdict)", "finding": "MMLU/HELM descriptions, pricing categories and SWE-bench boundary verify; hypothetical threshold and invoice example are labelled.", "check_ids": [ "B-model-benchmarking-01", "B-model-benchmarking-02", "B-model-benchmarking-03", "B-model-benchmarking-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/choosing-model-benchmark.md:86 (Turn the result into an operating decision)", "finding": "Invoice contract and numerical decision rule make selection concrete. A fully populated scorecard with an uncertainty-based gate would better demonstrate the decision.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/choosing-model-benchmark.md:49 (Decide what is inside the comparison)", "finding": "Readable numbered distinction and well-integrated contextual links; long paragraphs and recurring general reminders slightly dilute pace.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/choosing-model-benchmark.md:1 (front matter)", "finding": "Workload-specific benchmarking query and description match the guide; map reader question largely repeats the title but remains clear.", "check_ids": [] } ] }, { "id": "task-evaluation", "file": "articles/build-task-eval.md", "scores": { "accuracy_and_sources": 4.2, "practical_usefulness": 4.2, "writing_quality": 4.3, "intent_and_metadata": 4.5 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/build-task-eval.md:45 (Separate development from final evaluation)", "finding": "Grouping and holdout advice are sound, independently corroborated; no nearby primary citations despite several factual generalisation claims. Sample arithmetic is correct.", "check_ids": [ "B-task-evaluation-01", "B-task-evaluation-02", "B-task-evaluation-03" ] }, { "dimension": "practical_usefulness", "location": "B/articles/build-task-eval.md:69 (Example: a small first version)", "finding": "120-case example makes sampling and label repair tangible; lacks a filled case row and explicit ordinary-traffic weights for the mixed sample.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/build-task-eval.md:17 (Write a task contract)", "finding": "Five-part contract and concrete urgent-ticket consequences are clear; task, labels and audit record sections sometimes repeat the same instruction.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/build-task-eval.md:1 (front matter)", "finding": "Practical design intent is specific and fulfilled; title/description/query and stable ID agree across metadata and brief.", "check_ids": [] } ] }, { "id": "dataset-integrity", "file": "articles/datasets-holdouts-contamination.md", "scores": { "accuracy_and_sources": 3.9, "practical_usefulness": 4.1, "writing_quality": 4.2, "intent_and_metadata": 4.4 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/datasets-holdouts-contamination.md:25 (Understand different forms of leakage)", "finding": "2026 preprint is real and scoped accurately. The 2023 source is real but its method summary needs correction; see B-dataset-integrity-02.", "check_ids": [ "B-dataset-integrity-01", "B-dataset-integrity-02", "B-dataset-integrity-03" ] }, { "dimension": "practical_usefulness", "location": "B/articles/datasets-holdouts-contamination.md:53 (A contamination review you can run)", "finding": "Eight checks, access separation and version bridges are useful. A concrete duplicate/group audit example would strengthen implementation.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/datasets-holdouts-contamination.md:68 (Fresh examples are not automatically better)", "finding": "Balances freshness, representativeness and label quality clearly; contamination and general leakage are less sharply separated than they could be.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/datasets-holdouts-contamination.md:1 (front matter)", "finding": "Data-leakage query and description accurately promise split/provenance controls; metadata and map agree, without search-volume assertions.", "check_ids": [] } ] }, { "id": "evaluation-metrics", "file": "articles/metrics-rubrics.md", "scores": { "accuracy_and_sources": 4.3, "practical_usefulness": 4.2, "writing_quality": 4.3, "intent_and_metadata": 4.5 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/metrics-rubrics.md:41 (Validate automated metrics)", "finding": "Official grader and task-specific evaluation claims verify; precision/recall definitions are correct, but lack a nearby metrics reference.", "check_ids": [ "B-evaluation-metrics-01", "B-evaluation-metrics-02", "B-evaluation-metrics-03", "B-evaluation-metrics-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/metrics-rubrics.md:33 (Write rubrics like executable specifications)", "finding": "Good semantic/format distinction and selective-system coverage. Groundedness anchors overlap: level 2 permits minor unsupported claims under its material-only wording; needs disjoint anchors.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/metrics-rubrics.md:57 (Example scorecard for invoice extraction)", "finding": "Concrete scorecard is scannable and explains hard gates; no scored answer or numerical confusion-matrix walkthrough completes the demonstration.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/metrics-rubrics.md:1 (front matter)", "finding": "Metric/rubric intent, title and query match observable measures and scoring guidance; descriptions are accurate and fields agree.", "check_ids": [] } ] }, { "id": "human-judging", "file": "articles/human-evaluation-model-judges.md", "scores": { "accuracy_and_sources": 4.1, "practical_usefulness": 4.1, "writing_quality": 4.3, "intent_and_metadata": 4.5 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/human-evaluation-model-judges.md:31 (Reduce avoidable bias)", "finding": "Position-bias finding is supported by the linked paper, but Zheng author label is wrong. Other calibration guidance verifies; additional judge-bias claims need nearby citations.", "check_ids": [ "B-human-judging-01", "B-human-judging-02", "B-human-judging-03", "B-human-judging-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/human-evaluation-model-judges.md:73 (A deployment pattern)", "finding": "Human calibration, pass audits and revalidation are useful. Low-confidence routing needs a validated signal, and an actual confusion matrix would expose false-pass risk.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/human-evaluation-model-judges.md:61 (Example: answer grounded in a policy)", "finding": "Policy grounding example gives context and clear boundaries; broad agreement-statistic choices remain less concrete than the surrounding prose.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/human-evaluation-model-judges.md:1 (front matter)", "finding": "Human-versus-model judging question is explicit and fulfilled; title/query/description and stable ID align.", "check_ids": [] } ] }, { "id": "repeatability", "file": "articles/repeatability-uncertainty.md", "scores": { "accuracy_and_sources": 4.2, "practical_usefulness": 3.6, "writing_quality": 4.1, "intent_and_metadata": 4.3 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/repeatability-uncertainty.md:40 (Report intervals with their assumptions)", "finding": "Variance paper supports uncertainty and paired comparisons; no invented interval. Case/repeat arithmetic is correct and nesting is acknowledged.", "check_ids": [ "B-repeatability-01", "B-repeatability-02", "B-repeatability-03", "B-repeatability-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/repeatability-uncertainty.md:60 (A lightweight protocol)", "finding": "Tells readers to choose a suitable interval method but provides neither a computable example nor an analysis recipe. The 82/85 example cannot be analysed from supplied marginal totals alone.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/repeatability-uncertainty.md:75 (Example interpretation)", "finding": "Accessible distinction between observation and inference; repeated generic cautions and missing worked interval limit the article’s explanatory payoff.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/repeatability-uncertainty.md:1 (front matter)", "finding": "Query promises confidence intervals and repeatability; repeatability is answered well, but interval implementation and sample planning are only sketched.", "check_ids": [] } ] }, { "id": "fair-comparison", "file": "articles/fair-model-comparisons.md", "scores": { "accuracy_and_sources": 4.1, "practical_usefulness": 4.1, "writing_quality": 4.3, "intent_and_metadata": 4.5 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/fair-model-comparisons.md:23 (Freeze the surrounding system)", "finding": "Reasonable, independently corroborated prompt/interface and timing cautions; entirely uncited article leaves factual claims less traceable.", "check_ids": [ "B-fair-comparison-01", "B-fair-comparison-02" ] }, { "dimension": "practical_usefulness", "location": "B/articles/fair-model-comparisons.md:55 (Publish the configuration ledger)", "finding": "Controls tuning budget, tools, schema, ordering and system attribution. Ledger fields are useful, but there is no filled specimen or sensitivity experiment.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/fair-model-comparisons.md:49 (Example: summarising incident reports)", "finding": "Two-experiment example is clear and appropriately hypothetical; straightforward structure, with recurring boundary reminders at both ends.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/fair-model-comparisons.md:1 (front matter)", "finding": "Specific fair-comparison query and descriptive metadata accurately reflect the controls; all required fields align.", "check_ids": [] } ] }, { "id": "latency-cost", "file": "articles/latency-cost.md", "scores": { "accuracy_and_sources": 4.2, "practical_usefulness": 4.1, "writing_quality": 4.3, "intent_and_metadata": 4.4 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/latency-cost.md:55 (Example worksheet)", "finding": "Arithmetic $258/$127 is correct and hypothetical. Pricing categories verify. Cost equation’s retry term is ambiguous if input/output totals already include retries.", "check_ids": [ "B-latency-cost-01", "B-latency-cost-02", "B-latency-cost-03", "B-latency-cost-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/latency-cost.md:23 (Calculate cost transparently)", "finding": "Good human-review economics and latency boundaries, but worksheet omits per-success ratios and zero-success handling; mutually exclusive charge categories should be explicit.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/latency-cost.md:35 (Measure latency users feel)", "finding": "Plain operational language distinguishes first useful output and completion; short coherent sections, with a prose worksheet that would benefit from a table.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/latency-cost.md:1 (front matter)", "finding": "Latency and cost-per-task query is relevant, but description’s calculation promise is only partly completed in the worksheet; uppercase slug segment needs normalisation.", "check_ids": [] } ] }, { "id": "agent-evaluation", "file": "articles/agent-system-evaluation.md", "scores": { "accuracy_and_sources": 4.4, "practical_usefulness": 4.1, "writing_quality": 4.3, "intent_and_metadata": 4.4 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/agent-system-evaluation.md:37 (Control the environment)", "finding": "2026 benchmark-quality preprint and official trace guidance support the scoped claims. Real sources, with no invented performance figures.", "check_ids": [ "B-agent-evaluation-01", "B-agent-evaluation-02" ] }, { "dimension": "practical_usefulness", "location": "B/articles/agent-system-evaluation.md:45 (Score cost, risk and recovery)", "finding": "Provides coding sandbox, external tests, ablations and reset rules. Recovery is named but lacks an actual injected-failure/expected-recovery scenario.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/agent-system-evaluation.md:53 (Example: repository issue repair)", "finding": "Concrete repair setup separates outcome from narrative; clear flow, though the final link bundle repeats earlier cross-references.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/agent-system-evaluation.md:1 (front matter)", "finding": "Agent-system title and description correctly match trajectories and permissions; specific query fits, with uppercase AI slug segment an ordinary metadata edit.", "check_ids": [] } ] }, { "id": "evaluation-maintenance", "file": "articles/maintaining-evaluations.md", "scores": { "accuracy_and_sources": 4.4, "practical_usefulness": 4.2, "writing_quality": 4.3, "intent_and_metadata": 4.4 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/maintaining-evaluations.md:23 (Build a continuous evaluation loop)", "finding": "Continuous evaluation and growing monitored cases verify. Example schedule is explicitly illustrative, not an empirical or universal rule.", "check_ids": [ "B-evaluation-maintenance-01", "B-evaluation-maintenance-02" ] }, { "dimension": "practical_usefulness", "location": "B/articles/maintaining-evaluations.md:54 (Example release routine)", "finding": "40/300/200-case routine, ownership and bridge comparisons are actionable. A regression-triage record would better show how an actual release decision follows.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/maintaining-evaluations.md:36 (Refresh without breaking comparisons)", "finding": "Anchor/recent-set distinction and clear change-history prose make maintenance understandable; overlap with dataset article is noticeable.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/maintaining-evaluations.md:1 (front matter)", "finding": "Maintenance query and description fit the operating routine; all fields agree, with uppercase LLM slug segment for copy normalisation.", "check_ids": [] } ] }, { "id": "public-benchmarks", "file": "articles/public-benchmark-literacy.md", "scores": { "accuracy_and_sources": 3.8, "practical_usefulness": 4.0, "writing_quality": 4.2, "intent_and_metadata": 4.4 }, "evidence": [ { "dimension": "accuracy_and_sources", "location": "B/articles/public-benchmark-literacy.md:41 (Consider contamination and task exposure)", "finding": "MMLU and HELM scope verify. Correct the source-method summary and author label identified in B-public-benchmarks-03/04.", "check_ids": [ "B-public-benchmarks-01", "B-public-benchmarks-02", "B-public-benchmarks-03", "B-public-benchmarks-04" ] }, { "dimension": "practical_usefulness", "location": "B/articles/public-benchmark-literacy.md:53 (Turn a public score into a shortlist)", "finding": "Evidence-table fields and reading checklist are practical, but no real or fully populated illustrative benchmark card demonstrates application.", "check_ids": [] }, { "dimension": "writing_quality", "location": "B/articles/public-benchmark-literacy.md:33 (Understand the score, not just the rank)", "finding": "Useful task/metric/aggregation distinctions; prose is readable but overlaps the pillar’s shortlist cautions and closes with another checklist.", "check_ids": [] }, { "dimension": "intent_and_metadata", "location": "B/articles/public-benchmark-literacy.md:1 (front matter)", "finding": "Public-benchmark intent/title/query are well aligned; accurate description and consistent records, with uppercase AI slug segment needing routine normalisation.", "check_ids": [] } ] } ] } } }