{ "version": 1, "canonical_plan": "DOC-3519 v1.1", "classification": "Later editorial aids; teaching data are hypothetical, not benchmark results", "teaching_data": { "luna_uncertainty": { "classification": "Invented paired example already present in Luna publication copy", "n": 100, "paired_counts": { "both_pass": 80, "a_only": 2, "b_only": 5, "neither": 13 }, "a_successes": 82, "b_successes": 85, "b_minus_a": 0.03, "instruction": "Render the paired table. Preserve existing seed20261022/bootstrap code and assumptions in article; do not borrow Sol’s different paired counts or attach an interval to real editorial scores." }, "luna_cost": { "classification": "Invented assisted-workflow worksheet already present in the Luna article", "currency": "Illustrative dollars; the source does not specify AUD or USD. Do not imply current provider pricing.", "cases": 500, "review_minutes_per_case": 4, "illustrative_cost_per_review_minute": 1, "a": { "autonomous_successes": 440, "review_cases": 60, "source_api_and_tool_cost": 18, "review_minutes": 240, "total_assisted_cost": 258, "assisted_successes_if_all_repairs_succeed": 500, "cost_per_assisted_completion": 0.516, "autonomous_pass_rate": 0.88 }, "b": { "autonomous_successes": 475, "review_cases": 25, "source_api_cost": 27, "review_minutes": 100, "total_assisted_cost": 127, "assisted_successes_if_all_repairs_succeed": 500, "cost_per_assisted_completion": 0.254, "autonomous_pass_rate": 0.95 }, "limits": "All repairs succeeding is an explicit hypothetical assumption. Token/cache counts and median/p95 latency are not supplied. Zero accepted outcomes makes cost per success undefined; report spend and zero outcomes, not a finite ratio." } }, "original_source_hashes": { "src/assets/evidence/ai-editorial-20261022/article-scores.json": "edbfa6b9db723fa53f70f4ad4b6c75dad7da5d4989b50005e8185825b50a36c6", "src/assets/evidence/ai-editorial-20261022/blind-review.zip": "f5a2945ed4dabc295717253580cf4a6501468baa86eb81323c3f8e74a3d238ec", "src/assets/evidence/ai-editorial-20261022/cluster-scores.json": "2bcbc44deee544e2d71d6c2728c59bd8bf978cd32777fa78de925a3aee3d18b8", "src/assets/evidence/ai-editorial-20261022/fact-check-ledger.json": "4b469e6cfc1c2a990e1a094756270dd7acefb2a5191bd60042258a6519d39c25", "src/assets/evidence/ai-editorial-20261022/frozen-rubric.json": "dccb6015970577c821a01750fa2353c75893b74be95a5d3f3a4fcea8835294cc", "src/assets/evidence/ai-editorial-20261022/original-luna-submission.zip": "a227fdda4a7999b6840ee5dc20583accd98027f7e7c8de0687f079218c45840b", "src/assets/evidence/ai-editorial-20261022/original-sol-submission.zip": "24822bd993fe04b52232e862f3570edcf45d11ca5eab0629d64684767d1d2c4d", "src/assets/evidence/ai-editorial-20261022/original-source-sha256.json": "141b37cf8abc95b234f14d5b50e3a6182051393d24024aeea5379546c36bb218", "src/assets/evidence/ai-editorial-20261022/publication-edit-log.json": "6fd3c305d0ca1ac4c564664e45f7f2b41e00a707c3f5624b907f24eeadb34b0a", "src/assets/evidence/ai-editorial-20261022/shared-production-brief.txt": "0574b8270cced7ce5a5b3adcd131bc207c163d546fe3cd7bf3a6d76ea5973d88", "src/assets/evidence/ai-editorial-20261022/token-usage.json": "4cb2d2ea426e1ac74dd21143575fa46aa027ecc98d3ca18dc9792e7a473934ac", "src/assets/evidence/ai-editorial-deepseek-20261025/article-scores.json": "524be32d881e31f7918c95136db184270cdfd2924718f633d5351264c0991dfd", "src/assets/evidence/ai-editorial-deepseek-20261025/blind-decision.json": "f16941cccc342e12c2c6fa6fcd6bb58e663a766b04c30b31ea489d57bc503fad", "src/assets/evidence/ai-editorial-deepseek-20261025/blind-review.md": "0a66692d52725d7bd5633401efe783d91a150caaaff16472a2f4367ffa5cf081", "src/assets/evidence/ai-editorial-deepseek-20261025/cluster-scores.json": "044d95a070bf02a7025762ef861bbfd50c8a034bc617656206a2ef8dd273b90a", "src/assets/evidence/ai-editorial-deepseek-20261025/deepseek-request-usage.json": "3f7c09a0a0f1f12e0742d7da9f6cb77f81b8d22200c6ef892e87f02e59f06752", "src/assets/evidence/ai-editorial-deepseek-20261025/editing-plan.md": "876e91f3bb5929918a1d1d6ec43e3492fa350a277c38449fa9e0dc889a86ada3", "src/assets/evidence/ai-editorial-deepseek-20261025/efficiency-comparison.json": "7da877edbf9f9ecc3a372db23ff403bbec8001618cd6b065652127bb762169ad", "src/assets/evidence/ai-editorial-deepseek-20261025/export-notes.json": "af8a9a470c5a9f55f2f31aab7abee4f07a84397a1a68e0083c64a8e2efb2aed4", "src/assets/evidence/ai-editorial-deepseek-20261025/fact-check-ledger.json": "ae78810dd3244b451a23763f9226247374fc0280e597e1f5c71f0ad0383f34c2", "src/assets/evidence/ai-editorial-deepseek-20261025/identity-efficiency-release.json": "55faa08ee905268d053c95d94d282150f32bd4383aec1323c4657b5fd0df2573", "src/assets/evidence/ai-editorial-deepseek-20261025/judge-usage.json": "6e8a17442115c6ca0d5ae7916833efab8e31068ca916d0b46e79450f6bde9dc5", "src/assets/evidence/ai-editorial-deepseek-20261025/luna-request-usage.json": "829a9eba6b44bd9b5f90e605ec6fcd1e9ffb2532c798e6eaf757bfc69952f60a", "src/assets/evidence/ai-editorial-deepseek-20261025/manifest.json": "97e35e837136a1eba9f318b4b31c7999cee427508d39ee5fc73d35a627117f0e", "src/assets/evidence/ai-editorial-deepseek-20261025/original-deepseek-submission.zip": "95a2f07762c9dcce2f518b6d8d53eb5a033aab4162fad68182e73e077b686ea8", "src/assets/evidence/ai-editorial-deepseek-20261025/original-source-sha256.json": "e8b8cb7a2c07d335ebe8886a2e1422cd33d949cc25c3da88ffcdfea37c1d89b5", "src/assets/evidence/ai-editorial-deepseek-20261025/pairwise-comparison.json": "152ce3fb717c752d28f41c1a2ef30b491f52588b0b878ae3f80d2f383a24b657", "src/assets/evidence/ai-editorial-deepseek-20261025/production-brief.txt": "0574b8270cced7ce5a5b3adcd131bc207c163d546fe3cd7bf3a6d76ea5973d88", "src/assets/evidence/ai-editorial-deepseek-20261025/publication-edit-log.json": "cfe009163f88bd6ca139a0ff1467fb9794655d023b84d757387ed0f206e1fddf", "src/assets/evidence/ai-editorial-deepseek-20261025/quality-freeze-manifest.json": "2bf54cc6dfa6b103f70507ca284ba1aea244da6bda0241baa61ed7bfdc044193", "src/assets/evidence/ai-editorial-deepseek-20261025/rubric.json": "dccb6015970577c821a01750fa2353c75893b74be95a5d3f3a4fcea8835294cc", "src/assets/evidence/ai-editorial-deepseek-20261025/sol-request-usage.json": "6ea3f878920bc4a787ca01c39f605e8b8da90d6f5da42cf2eafe1ca92a88c3bb", "src/assets/evidence/ai-editorial-deepseek-20261025/three-way-review-request-usage.json": "bb9b0c53f42ac0e9e4e38803d0ec7748e97ac32a5c2bdc8baae4c5dbf76d7fbd", "src/assets/evidence/ai-editorial-deepseek-20261025/token-cost-estimates.json": "36dabfa44cc8d78ed9acdfa0c48899f13c742b79aaab81d39979a978379fbc5c", "src/assets/evidence/ai-editorial-deepseek-20261025/validation-report.json": "30929413b6f38539f488bd725c91c9406404fdc87fe6f074658ec101add271f9", "src/assets/evidence/ai-editorial-visuals-20261026/publication.json": "1ef4fabddf43087815cbcad1e91349ff2447b7aeb17d1d0169443660b9279c35", "src/assets/evidence/benchmark-guide-20261027/editorial-additions.json": "95800aca07418e315ef2b5eb5ee1ef82cbf5fc7e07c7fef6e2dab11bca33997e", "src/assets/evidence/benchmark-guide-20261027/visual-data.json": "eec2cc710bc2c00d389ea3efe7a6622261ac8f1a27c3919b6365a6961e490ceb" }, "illustrations": { "V01": { "title": "Grouped cases, three distinct evaluation roles", "kind": "split", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/ai-benchmark-datasets-holdouts-leakage/", "section": "Treat holdouts as a finite resource", "caption": "Illustrative dataset boundaries. Related cases stay grouped; development and protected evaluation serve different purposes.", "note": "Three roles follow this article. Limited data can constrain the split; state that limitation. Inspection that informs tuning changes a case’s role. Apply a declared refresh/reuse policy rather than an automatic retirement rule.", "steps": [ [ "Record provenance", "Identify source documents, conversations, entities, templates and collection periods." ], [ "Group related cases", "Keep the unit that could leak together; choose a split matching the generalisation question." ], [ "Assign declared roles", "Separate development access from candidate selection and the protected final comparison." ], [ "Freeze, evaluate, log exposure", "Freeze finalists and scorer before final evaluation; follow the declared exposure and reuse policy." ] ], "branches": [ [ "Development", "Prompt, rubric and code iteration are permitted; document versions." ], [ "Validation", "Candidate selection or threshold tuning; log access." ], [ "Final holdout", "Predeclared comparison or release gate; protected labels and recorded exposure." ] ], "source_sha256_before_addition": "17e3f4dbcd764081908ba26cc47522cc0b6898fcc2c8fe420c421d386370a913" }, "V02": { "title": "One hundred paired cases: where A and B disagree", "kind": "table", "classification": "Hypothetical paired example already present in this article", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/ai-benchmark-repeatability-uncertainty/", "section": "A computable paired illustration", "caption": "Calculated or invented teaching data, explicitly labelled; not a confidence interval for the editorial model ranking.", "note": "A passes 82/100; B passes 85/100; B−A is3 percentage points. Only seven cases are discordant. These are Luna’s hypothetical counts, not Sol’s different example or actual editorial scores. Existing bootstrap code below assumes independent, equally weighted cases; group dependent observations at the appropriate unit.", "headings": [ "Joint outcome on the same case", "Cases" ], "table": [ [ "Both pass", 80 ], [ "Only A passes", 2 ], [ "Only B passes", 5 ], [ "Neither passes", 13 ] ], "source_sha256_before_addition": "091d9967e990175f910b612608637be75d2c93171fc19442f19b6f1923a247a7" }, "V03": { "title": "Invoice extraction: a scorecard specification", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/ai-evaluation-metrics-rubrics/", "section": "Example scorecard for invoice extraction", "caption": "A proposed scorecard structure. The team supplies task-specific thresholds and validates its scorer.", "note": "Read this design together with the article’s assumptions; no sample sizes, thresholds or measured results are supplied by this aid.", "headings": [ "Observable requirement", "Scoring method / anchor", "Denominator / unit", "Gate or preference to declare", "Limitation" ], "table": [ [ "Correct field values", "Exact match after declared normalisation", "Each labelled field", "Declare critical fields before testing", "Normalisation must not erase currency or other meaningful distinctions." ], [ "Every critical field correct", "All required critical fields pass", "Invoices, including failed attempts", "Risk-based critical correctness gate", "Choose the non-inferiority margin from business risk; none supplied here." ], [ "No unsupported values", "Flag invented values when source is unreadable", "Unreadable-source cases; also show count", "Declare critical-error tolerance", "A plausible value is not source evidence." ], [ "Correct abstention / escalation", "Adjudicate permitted response", "Cases requiring abstention; also report coverage", "Declare acceptable escalation policy", "Abstaining on everything can hide poor coverage." ], [ "Valid structured output", "Schema and required-field checks", "All requested outputs", "Declare format gate", "Malformed and blank outputs remain in the denominator." ], [ "Response latency", "Measure p50 and p95 under declared harness", "Timed requests, including declared retry policy", "Declare operational preference or limit", "No latency measurements supplied." ], [ "Cost per successful invoice", "Total included spend / accepted invoices", "Successful invoices, with retries included", "Compare candidates passing critical gates", "Zero accepted invoices makes the ratio undefined; no costs supplied." ] ], "source_sha256_before_addition": "0037f96112596da8fdc67d9633b3621aac9362fe0145b4de99220f50ea9a1381", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V04": { "title": "Autonomous outcomes and assisted-workflow cost", "kind": "table", "classification": "Hypothetical worksheet — illustrative dollars, not provider pricing", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/ai-model-latency-cost-evaluation/", "section": "Example worksheet", "caption": "Hypothetical worksheet from the article, not measured provider performance or current pricing. Including assumed review effort changes the assisted-workflow comparison.", "note": "Assume 500 cases, four review minutes per reviewed case and one illustrative dollar per minute. Assisted outcomes reach 500 only if every repair succeeds; they are not autonomous successes. A lists API + tool cost 18; B lists API cost 27, with tool charges unreported, not measured zero. Currency is not specified as AUD or USD. Token/cache counts and median/p95 latency are unavailable. If accepted outcomes are zero, report spend and zero outcomes; cost per success is undefined.", "tables": [], "source_sha256_before_addition": "4e94ade591d1c0582a5f9d0d1fa91bf3eaa91b58d06829c38931016b76e042d3", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V05": { "title": "From a workload decision to monitored evidence", "kind": "flow", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/benchmark-ai-models-for-your-workload/", "section": "Start with the decision, not the leaderboard", "caption": "A proposed evaluation workflow: define the decision, protect the evidence, compare declared configurations and retain a reviewable decision.", "note": "Revision loop: monitoring → versioned workload/case specification or scorer review → a newly frozen candidate. This is not silent reuse of an exposed final holdout. Gates and thresholds must come from the actual business decision.", "steps": [ [ "Name the decision and gates", "Define success, unacceptable errors and the current baseline." ], [ "Specify workload and cases", "Record intended inputs, outputs, constraints and outcome oracle." ], [ "Group and split cases", "Separate development from protected evaluation at the relevant dependence unit." ], [ "Freeze finalists and scorer", "Record prompts, tools, versions, scoring rules and permitted attempts." ], [ "Run paired comparisons", "Use the same cases and declared protocol; retain failures and missing outputs." ], [ "Review quality, cost and failures", "Consider uncertainty, practical differences and consequential errors." ], [ "Decide and monitor", "Document the choice and limitations; monitoring triggers a new version and a reviewed evaluation plan." ] ], "source_sha256_before_addition": "3a2d04ca689abb4bb0f26f886f1996c4ce52ac4a6283ac27737184635cac764e" }, "V06": { "title": "Ticket classification coverage to plan before the pilot", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/design-ai-evaluation/", "section": "Build a benchmark matrix", "caption": "Illustrative coverage design, not collected test results or a mandated sample size.", "note": "A proposed coverage matrix, not observed pass rates or a production taxonomy. Include cases in a pilot or document why a combination is not applicable. Report deployment-weighted results separately from risk-focused challenge slices; no counts or prevalence are invented.", "headings": [ "Named case family", "Ordinary case", "Missing / ambiguous information", "Contradictory evidence", "Permitted next step", "Oracle", "Sampling role" ], "table": [ [ "Common labels / escalation routes", "Include in pilot: routine tickets", "Include in pilot: incomplete request", "Include in pilot: conflicting routing cues", "Apply declared label or escalation policy", "Adjudicated label and permitted route", "Representative sample; traffic weights require measured prevalence." ], [ "Mixed-intent messages", "Include in pilot: combined requests", "Include in pilot: unclear primary intent", "Include in pilot: incompatible intents", "Follow declared multi-label / escalation rule", "Adjudicated intent set and route", "Representative if observed; separate deliberate challenges." ], [ "Empty / malformed input", "Not an ordinary valid request; record reason", "Include in pilot: unusable input", "Include only where case makes sense; otherwise record N/A reason", "Ask for required information or reject per policy", "Input-validity and recovery contract", "Challenge slice; do not invent traffic share." ], [ "Hostile embedded instructions", "Ordinary surrounding ticket as control", "Include in pilot: missing trusted context", "Include in pilot: instructions conflict with policy", "Treat untrusted content under declared policy", "Permitted action and policy checks", "Deliberate challenge reported separately." ], [ "Out-of-scope requests", "Ordinary in-scope case as control", "Include in pilot: unclear scope", "Include in pilot: scope conflict", "Use declared refusal / escalation route", "Scope rubric and permitted next step", "Separate coverage and challenge reporting." ] ], "source_sha256_before_addition": "50ad575904acb84e0ae1307c4683c84f3118cda62413aa3e47a73529a17fea21", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V07": { "title": "Agent outcome verification and safe recovery", "kind": "recovery", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/evaluate-ai-agent-tool-use/", "section": "Define task success outside the agent", "caption": "An illustrative action-recovery test. A confident final message is not proof that the intended state change happened.", "note": "Illustrative control flow, not proof that a historical agent followed it. A tool invocation alone is not task success. Recovery depends on the application’s actual idempotency and state-reconciliation contract; an unknown result is not permission to duplicate a write.", "steps": [ [ "State the authorised outcome", "Record requested change, permissions, constraints and success oracle." ], [ "Issue the agent/tool operation", "Capture operation identity and the declared retry/recovery contract." ], [ "Receive response or timeout", "A plausible message or timeout does not establish the external outcome." ], [ "Read authoritative state", "Reconcile the operation/result in the system that owns it before another write." ] ], "branches": [ [ "Confirmed success", "Verify the outcome oracle and report the completed state." ], [ "Confirmed failure", "Follow the declared safe recovery path; retry only when the application contract permits it." ], [ "Still unknown", "Reconcile or escalate. Do not blindly repeat a possibly completed write." ] ], "source_sha256_before_addition": "74d79c6ad04878ab45573466e3053c5b98cbb1e44d702214e287fd8d938b7198" }, "V14": { "title": "Verify external outcomes before recovery", "kind": "recovery", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/evaluating-agents-and-tool-use/", "section": "Outcome oracles", "caption": "An illustrative action-recovery test. A confident final message is not proof that the intended state change happened.", "note": "Illustrative control flow, not proof that a historical agent followed it. A tool invocation alone is not task success. Recovery depends on the application’s actual idempotency and state-reconciliation contract; an unknown result is not permission to duplicate a write.", "steps": [ [ "State the authorised outcome", "Record requested change, permissions, constraints and success oracle." ], [ "Issue the agent/tool operation", "Capture operation identity and the declared retry/recovery contract." ], [ "Receive response or timeout", "A plausible message or timeout does not establish the external outcome." ], [ "Read authoritative state", "Reconcile the operation/result in the system that owns it before another write." ] ], "branches": [ [ "Confirmed success", "Verify the outcome oracle and report the completed state." ], [ "Confirmed failure", "Follow the declared safe recovery path; retry only when the application contract permits it." ], [ "Still unknown", "Reconcile or escalate. Do not blindly repeat a possibly completed write." ] ], "source_sha256_before_addition": "93b9df229c0c2e3a7f36fc548ef08e78ffafa3540bd2f78e85e79e2909b52fc9" }, "V08": { "title": "Configuration ledger for a fair comparison", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/fair-ai-model-comparison/", "section": "Publish the configuration ledger", "caption": "Declared comparison conditions. A model-only substitution and a tuned-system comparison support different conclusions.", "note": "Template for a proposed comparison, not a completed ledger for the historical editorial benchmark. No actual settings, matched compute, provider snapshot or 1017 identity are inferred. Record unavailable values explicitly; distinguish model and whole-system comparisons.", "headings": [ "Control", "Common rule to declare", "Candidate-specific record", "What inference it supports / limits" ], "table": [ [ "Prompt / context", "Common task requirements and available evidence", "Record each exact prompt and context construction", "Supports the declared system comparison, not a claim that prompts are identical." ], [ "Tools / retrieval", "Common permitted information and authority", "Record tool versions, corpus, retrieval and per-candidate support", "Differences can affect system outcomes." ], [ "Effort semantics", "Declare the comparison objective and resource policy", "Record provider-specific settings and unavailable options", "Nominal “high” does not establish matched computation." ], [ "Attempts / timeouts", "Predeclare retry, stop and timeout rules", "Record actual supported limits and usage", "A missing value is unavailable, not zero." ], [ "Development budget", "Declare allowed tuning opportunity", "Record candidate-specific iterations and resources", "Unequal tuning limits a model-only inference." ], [ "Scorer", "Same declared outcome definition and rubric", "Record scorer/version, calibration and exceptions", "Model judgement still needs validation." ], [ "Provider / version", "Record the identity each service actually exposes", "Record exact identifier, snapshot if verifiable, otherwise unknown", "A moving latest alias does not prove a pinned snapshot." ], [ "Research / run date", "State when evidence was collected", "Record dates and relevant configuration changes", "No current ranking or contemporaneous matching is asserted." ] ], "source_sha256_before_addition": "9b284ce665947b4231a6e18b04bf37b8113b6243a725c38d4bdd0e09bc8076a7", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V20": { "title": "A comparison manifest to complete before running", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/prompt-tool-and-configuration-fairness/", "section": "The configuration manifest", "caption": "Declared comparison conditions. A model-only substitution and a tuned-system comparison support different conclusions.", "note": "Template for a proposed comparison, not a completed ledger for the historical editorial benchmark. No actual settings, matched compute, provider snapshot or 1017 identity are inferred. Record unavailable values explicitly; distinguish model and whole-system comparisons.", "headings": [ "Control", "Common rule to declare", "Candidate-specific record", "What inference it supports / limits" ], "table": [ [ "Prompt / context", "Common task requirements and available evidence", "Record each exact prompt and context construction", "Supports the declared system comparison, not a claim that prompts are identical." ], [ "Tools / retrieval", "Common permitted information and authority", "Record tool versions, corpus, retrieval and per-candidate support", "Differences can affect system outcomes." ], [ "Effort semantics", "Declare the comparison objective and resource policy", "Record provider-specific settings and unavailable options", "Nominal “high” does not establish matched computation." ], [ "Attempts / timeouts", "Predeclare retry, stop and timeout rules", "Record actual supported limits and usage", "A missing value is unavailable, not zero." ], [ "Development budget", "Declare allowed tuning opportunity", "Record candidate-specific iterations and resources", "Unequal tuning limits a model-only inference." ], [ "Scorer", "Same declared outcome definition and rubric", "Record scorer/version, calibration and exceptions", "Model judgement still needs validation." ], [ "Provider / version", "Record the identity each service actually exposes", "Record exact identifier, snapshot if verifiable, otherwise unknown", "A moving latest alias does not prove a pinned snapshot." ], [ "Research / run date", "State when evidence was collected", "Record dates and relevant configuration changes", "No current ranking or contemporaneous matching is asserted." ] ], "source_sha256_before_addition": "fc92132267ccac97423c94ebaac3cd9627b2d014285666d340e4a8eb9846e56f", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V09": { "title": "Validate the judge before trusting the score", "kind": "flow", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/human-evaluation-llm-as-judge/", "section": "A deployment pattern", "caption": "A proposed grading workflow. Agreement alone does not establish detection of consequential failures.", "note": "Reference labels are adjudicated judgements, not infallible ground truth. Keep calibration and held-apart validation distinct. No universal sample minimum, acceptable error threshold or measured judge accuracy is supplied.", "steps": [ [ "Draft the rubric", "Define observable anchors and consequential errors before scaling." ], [ "Label and adjudicate development cases", "Collect independent judgements; discuss disagreements and retain their rationale." ], [ "Calibrate anchors", "Revise ambiguous criteria on development cases rather than the validation answers." ], [ "Validate blind and vary order", "Use held-apart judgements; hide candidate identity and test ordering effects." ], [ "Inspect consequential disagreements", "Examine false passes and missed failures, not only aggregate agreement." ], [ "Escalate to a human", "Use explicit risk/disagreement routes; model self-confidence alone is not validated error detection." ], [ "Version and monitor", "Record rubric/judge changes and repeat validation when relevant evidence changes." ] ], "source_sha256_before_addition": "662034032a7b8a4bbe392cbb4e99d9d858b29a59e2fad24e77191cabd6906d87" }, "V16": { "title": "A validation protocol for human and model judgements", "kind": "flow", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/human-evaluation-and-model-judges/", "section": "A validation protocol", "caption": "A proposed grading workflow. Agreement alone does not establish detection of consequential failures.", "note": "Reference labels are adjudicated judgements, not infallible ground truth. Keep calibration and held-apart validation distinct. No universal sample minimum, acceptable error threshold or measured judge accuracy is supplied.", "steps": [ [ "Draft the rubric", "Define observable anchors and consequential errors before scaling." ], [ "Label and adjudicate development cases", "Collect independent judgements; discuss disagreements and retain their rationale." ], [ "Calibrate anchors", "Revise ambiguous criteria on development cases rather than the validation answers." ], [ "Validate blind and vary order", "Use held-apart judgements; hide candidate identity and test ordering effects." ], [ "Inspect consequential disagreements", "Examine false passes and missed failures, not only aggregate agreement." ], [ "Escalate to a human", "Use explicit risk/disagreement routes; model self-confidence alone is not validated error detection." ], [ "Version and monitor", "Record rubric/judge changes and repeat validation when relevant evidence changes." ] ], "source_sha256_before_addition": "950207bc9c9d4569863daa673349f067e9904b8395e3e1b84a4cc65019db1669" }, "V10": { "title": "Turn monitored failures into versioned evaluation coverage", "kind": "maintenance", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/maintain-llm-evaluations/", "section": "Build a continuous evaluation loop", "caption": "A proposed maintenance loop with versioned cases and decisions; adding data does not erase earlier results.", "note": "Revision loop: monitoring → triage → a new case/version → a newly frozen candidate. Do not silently promote an exposed case back into independent final evidence. Cadence and sample sizes depend on workload and risk; none are mandated here.", "steps": [ [ "Observe an incident or change", "Capture affected task, model, tool, data or policy and the evidence." ], [ "Triage the failure component", "Distinguish model output, retrieval, orchestration, permissions, scorer and environment." ], [ "Curate and adjudicate a case", "Minimise sensitive data and establish the expected permitted outcome." ], [ "Version development / regression coverage", "Keep provenance and change history; do not rewrite past score series." ], [ "Test a frozen candidate", "Run the declared suite and inspect consequential failures and uncertainty." ], [ "Record the release decision", "Document configuration, results, limitations and follow-up obligations." ], [ "Monitor after the decision", "New incidents return to triage and a versioned evaluation plan." ] ], "branches": [ [ "Development / regression layer", "Diagnose known incidents and test fixes against versioned coverage." ], [ "Protected holdout layer", "Keep access, labels, exposure and reuse under a separately declared final-evaluation policy." ] ], "source_sha256_before_addition": "9c8a83f1e5d854e5b54815c5860bdde7afbbf685b5b56c078be5528330ce0d9f" }, "V17": { "title": "Maintain an evaluation without rewriting its history", "kind": "maintenance", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/interpreting-results-and-maintaining-evals/", "section": "Maintaining the evaluation", "caption": "A proposed maintenance loop with versioned cases and decisions; adding data does not erase earlier results.", "note": "Revision loop: monitoring → triage → a new case/version → a newly frozen candidate. Do not silently promote an exposed case back into independent final evidence. Cadence and sample sizes depend on workload and risk; none are mandated here.", "steps": [ [ "Observe an incident or change", "Capture affected task, model, tool, data or policy and the evidence." ], [ "Triage the failure component", "Distinguish model output, retrieval, orchestration, permissions, scorer and environment." ], [ "Curate and adjudicate a case", "Minimise sensitive data and establish the expected permitted outcome." ], [ "Version development / regression coverage", "Keep provenance and change history; do not rewrite past score series." ], [ "Test a frozen candidate", "Run the declared suite and inspect consequential failures and uncertainty." ], [ "Record the release decision", "Document configuration, results, limitations and follow-up obligations." ], [ "Monitor after the decision", "New incidents return to triage and a versioned evaluation plan." ] ], "branches": [ [ "Development / regression layer", "Diagnose known incidents and test fixes against versioned coverage." ], [ "Protected holdout layer", "Keep access, labels, exposure and reuse under a separately declared final-evaluation policy." ] ], "source_sha256_before_addition": "13d19db736917a6aa9f007d4dac5f7d68224331b74738552c9a4b47e9b08a426" }, "V11": { "title": "An evidence card for reading public benchmark claims", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-2026-10-22/luna/read-public-ai-benchmarks/", "section": "A quick reading checklist", "caption": "A benchmark-reading template; a public score screens candidates rather than proving fitness for this workload.", "note": "Read this design together with the article’s assumptions; no sample sizes, thresholds or measured results are supplied by this aid.", "headings": [ "Field to inspect", "Question to record", "Limit on the conclusion" ], "table": [ [ "Task / version", "What task, dataset version and model/service identity were evaluated?", "A task score is not overall model intelligence." ], [ "Score / denominator", "What was counted, excluded and aggregated, and over how many units?", "A percentage without counts conceals sampling limits." ], [ "Harness / tools / attempts", "What context, tools, effort, retry and time budgets were available?", "Different systems may not isolate the base model." ], [ "Sample / coverage", "Which population, slices and difficult cases were covered?", "Do not assume relevance to your local workload." ], [ "Grader", "Who or what scored, against which anchors and validation?", "Agreement alone can miss consequential false passes." ], [ "Data exposure", "What contamination checks and training-exposure limits are known?", "Unverified exposure remains unknown." ], [ "Local workload question", "What decision-relevant test remains to run for your own users?", "Use public evidence to shortlist; local suitability is unresolved." ] ], "source_sha256_before_addition": "687b33e83922901a06747ea2dc89dbd032c3f68e07cc913f35e5f58f8ac4b836" }, "V12": { "title": "Keep development feedback outside the protected comparison", "kind": "split", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/contamination-leakage-and-holdouts/", "section": "Controls that actually help", "caption": "Illustrative dataset boundaries. Related cases stay grouped; development and protected evaluation serve different purposes.", "note": "Group related conversations, templates, source records or time units before splitting. Keep answer-bearing metadata and retrieval out of candidate context. No contamination check proves unseen training data, and no blanket automatic-retirement rule is added by this diagram.", "steps": [ [ "Record provenance", "Identify source documents, conversations, entities, templates and collection periods." ], [ "Group related cases", "Keep the unit that could leak together; choose a split matching the generalisation question." ], [ "Assign declared roles", "Separate development access from candidate selection and the protected final comparison." ], [ "Freeze, evaluate, log exposure", "Freeze finalists and scorer before final evaluation; follow the declared exposure and reuse policy." ] ], "branches": [ [ "Development / calibration", "Inspect and tune against declared development material; log changes." ], [ "Protected evaluation", "Keep inputs and answer keys separate; freeze candidates and scorer before final comparison." ], [ "Exposure / reuse record", "Record who saw what and when; apply the stated reuse or refresh policy." ] ], "source_sha256_before_addition": "324f8f859ad4b0f3cdb4d3f8afba5b8bed005b5588721496f2fe5c9b013a1d63" }, "V13": { "title": "Workload task types and conditions to cover", "kind": "table", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/designing-evaluation-tasks-and-datasets/", "section": "The sampling frame and stratification", "caption": "Illustrative coverage design, not collected test results or a mandated sample size.", "note": "A proposed matrix using this article’s extract/classify/summarise/answer/act families. Include combinations in a pilot, or record an N/A reason. Frequency, difficulty, risk and format are strata to document; no realised sample size, prevalence or pass rate is supplied.", "headings": [ "Task type", "Ordinary case", "Missing / ambiguous information", "Contradictory evidence", "Permitted next step", "Oracle", "Sampling role" ], "table": [ [ "Extract", "Include in pilot: clear source fields", "Include in pilot: unreadable / absent field", "Include in pilot: conflicting source values", "Return supported fields or declared escalation", "Field reference and schema; adjudicate ambiguity", "Representative and deliberate challenges reported separately." ], [ "Classify", "Include in pilot: clear class", "Include in pilot: mixed / unclear intent", "Include in pilot: conflicting signals", "Use declared class or escalation rule", "Label guide and adjudicated reference", "Record measured prevalence before traffic weighting." ], [ "Summarise", "Include in pilot: coherent source", "Include in pilot: incomplete context", "Include in pilot: inconsistent statements", "Ground summary in source and expose uncertainty", "Anchored factuality / coverage rubric", "Stratify difficulty, risk and format; no fabricated counts." ], [ "Answer", "Include in pilot: supported question", "Include in pilot: missing evidence", "Include in pilot: conflicting sources", "Answer from permitted evidence or abstain", "Reference plus rubric / source checks", "Separate challenge sampling from deployment estimates." ], [ "Act", "Include in pilot: authorised operation", "Include in pilot: missing authority / target", "Include in pilot: conflicting requested state", "Clarify, refuse or act within declared permissions", "Authoritative final state and constraints", "Risk-focused cases are not estimated traffic share." ] ], "source_sha256_before_addition": "50c551eb4c653306291790d92c618f1cc442cdc0d23efac380d141f8a219d375", "scroll_hint": "Wide table: scroll horizontally to read every column. Keyboard users can focus the table area and use the arrow keys." }, "V15": { "title": "Six steps from evaluation decision to monitored operation", "kind": "flow", "classification": "Illustrative evaluation design — not measured model performance", "article_url": "/benchmarks/ai-editorial-deepseek-2026-10-25/deepseek/evaluating-ai-models-and-agent-systems/", "section": "A six-step evaluation loop", "caption": "The article’s proposed six-step evaluation loop, not evidence that a particular deployment executed these controls.", "note": "Revision paths: an inadequate sample returns to step 2; an unreliable scorer returns to step 3; monitoring can trigger a newly scoped evaluation. Exactly six stages follow the article. An example error threshold elsewhere is not a universal business rule.", "steps": [ [ "Name the decision and threshold", "State the business choice and acceptable error from its actual risk." ], [ "Build a workload-resembling sample", "Record provenance, population, strata and coverage limitations." ], [ "Choose and calibrate a scorer", "Define the oracle/rubric and validate it before trusting scores." ], [ "Design and configure the run", "Freeze configurations, budgets, trials and collection rules." ], [ "Analyse the evidence", "Review quality, uncertainty, cost, latency and failure modes together." ], [ "Decide, document and monitor", "Record the choice and limits; monitor changes and outcomes." ] ], "source_sha256_before_addition": "71b96247381723a71b060cda2a4cef8a02e6664907a189cf4fd07a86c2949cb1" } } }