{ "version": 1, "prepared_on": "2026-10-27", "purpose": "Later editorial illustrations for the local M2 preview. Frozen contestant submissions and historical scores are unchanged. Public release is separately governed.", "illustrations": { "V24": { "title": "From a business decision to a reviewable model choice", "kind": "flow", "article_url": "/blog/ai-model-benchmarking/", "section": "Start with a decision contract", "source_sha256_before_addition": "e14b3805c776aac27fc13bc8a9e8082ffb278b2c97b7fd1e6b6db7483b68a306", "caption": "A proposed evaluation workflow: define the decision, protect the evidence, compare declared configurations and retain a reviewable decision.", "classification": "Proposed evaluation workflow · not a measured experiment", "steps": [ [ "Decision and gates", "Name the choice, required outcomes and unacceptable failures." ], [ "Workload and cases", "Specify inputs, context, expected behaviour and permitted actions." ], [ "Group and protect data", "Keep related cases together; separate development from the protected holdout." ], [ "Freeze the comparison", "Declare finalists, prompts, tools, budgets and scorer before final evaluation." ], [ "Run paired cases", "Compare declared configurations on the same independent evaluation units." ], [ "Review quality, cost and failures", "Keep hard gates, estimates, measured outcomes and missing values distinct." ], [ "Record the decision", "Retain the evidence, remaining risks and the person accepting them." ], [ "Monitor and version the next test", "New incidents can start a new version; do not silently reuse an exposed holdout." ] ], "note": "The numbered order is the diagram’s text equivalent. Monitoring feeds a declared new evaluation version, not an unrecorded return to the same holdout." }, "V30": { "title": "Two review rounds; preserved originals", "kind": "rounds", "article_url": "/blog/how-we-benchmark-ai-models-for-editorial-work/", "section": "Review quality before revealing speed and usage", "source_sha256_before_addition": "02198baeaec19d1d5aa45191d69203fdb5bce258874b7c4417836c0167b9b9d9", "caption": "The recorded editorial review sequence, with original and extension rounds kept distinct.", "classification": "Recorded historical sequence · later editorial presentation", "stages": [ [ "Shared commission", "Shared brief and frozen rubric", "Same brief and rubric; original Sol/Luna submissions retained" ], [ "Production", "Independent Sol and Luna sessions, 22 October", "DeepSeek extension, 25 October; different provider and CLI" ], [ "Evidence preservation", "Original files preserved and checksummed", "New submission preserved; originals not regenerated" ], [ "Anonymous quality review", "A = Sol; B = Luna, revealed after quality freeze", "A = Luna; B = Sol; C = DeepSeek, revealed after quality freeze" ], [ "Quality freeze", "Original two-model scores frozen", "Fresh three-way scores frozen; not a revision of the original scores" ], [ "Identity and efficiency", "Reveal producer identities; attach recorded production usage", "Reveal new round’s identities; attach recorded production usage" ], [ "Publication copies", "Disclosed editorial corrections; original downloads retained", "Disclosed extension and later figures; scored submissions stay unchanged" ] ], "note": "Both rounds used a single GPT-6.1 Sol model judge, sharing the winning producer’s exact model designation. A separate session and order checks are not independent human validation. This diagram records history; it does not claim proposed future controls were performed." }, "V25": { "title": "What 80 successes out of 100 can tell us", "kind": "uncertainty", "article_url": "/blog/benchmark-uncertainty-repeatability/", "section": "Put a single success rate in context", "source_sha256_before_addition": "cc6a83ff678682aabbf503659d11a9d01427238b1ef6299d196ab218ebf3b10c", "caption": "Calculated or invented teaching data, explicitly labelled; not a confidence interval for the editorial model ranking.", "classification": "Calculated teaching example · not the editorial benchmark", "note": "The 95% Wilson interval assumes independent, similarly sampled binary tasks and z = 1.96. It does not account for biased selection, related conversations or judge drift. The paired example is separate: its counts describe joint outcomes, not this interval’s endpoints." }, "V31": { "title": "A judge that passes everything misses every failure", "kind": "confusion", "article_url": "/blog/human-reviewers-llm-judges/", "section": "Measure errors that affect the decision", "source_sha256_before_addition": "8eada10cf44430b0c863cf29a90200af284ef05d1ed38cb45bda22401dd3e487", "caption": "Hypothetical teaching example from this article; these are not measured contestant results.", "classification": "Hypothetical human-label validation set · positive class = human-adjudicated pass", "dataset": "sol_judge", "positive": "Human-adjudicated pass", "negative": "Human-adjudicated failure", "predicted_positive": "Judge says pass", "predicted_negative": "Judge says fail", "note": "Agreement is 90/100 = 90%, yet failure-detection recall is 0/10 = 0%. There are no predicted failures, so precision for a failure prediction is undefined, not zero. This constructed audit composition is not a measured traffic failure rate." }, "V32": { "title": "Escalation counts behind the headline accuracy", "kind": "confusion", "article_url": "/blog/llm-evaluation-metrics-rubrics/", "section": "Work through a classification example", "source_sha256_before_addition": "95f07c2fafdde07744893d5e3755bd25c5ea6fba92d1068665fb52c9e6b481ba", "caption": "Hypothetical teaching example from this article; these are not measured contestant results.", "classification": "Hypothetical routing test · positive class = requires escalation", "dataset": "sol_metrics", "positive": "Requires escalation", "negative": "Does not require escalation", "predicted_positive": "Flagged for escalation", "predicted_negative": "Not flagged", "note": "Accuracy is 85%, but five of the twenty required escalations are missed. The operational cost of false negatives and false positives determines which metric matters. A zero denominator makes a ratio undefined; do not substitute zero." }, "V26": { "title": "Plan coverage before collecting a headline score", "kind": "table", "article_url": "/blog/design-llm-evaluation-tasks/", "section": "Build a coverage matrix", "source_sha256_before_addition": "1382b5452448cba927f51203496eb1892ffe3c7aefd847d6e136a8f890210981", "caption": "Illustrative coverage design, not collected test results or a mandated sample size.", "classification": "Illustrative coverage design · no measured counts or pass rates", "headings": [ "Task family", "Ordinary case", "Missing or ambiguous", "Contradictory evidence", "Expected next step / permitted action", "Oracle and coverage role" ], "table": [ [ "Eligibility question", "Include: clear policy and order", "Include: ask for missing purchase date", "Include: conflicting policy versions", "Answer only from the supplied applicable policy; escalate unresolved conflict", "Adjudicated policy decision; representative sample + deliberate challenges" ], [ "Return instructions", "Include: supported return route", "Include: ask for missing item detail", "Include: instructions conflict with order state", "Give supported instructions; do not claim a return was created", "Current policy and tool contract; representative sample + challenges" ], [ "Order-status explanation", "Include: authoritative status supplied", "Include: request an order identifier", "Include: stale note conflicts with current state", "Explain verified status; reconcile conflict before an action claim", "Authoritative order record; representative sample + challenges" ], [ "Unsupported request", "Include: request outside capability", "Include: clarify what is being requested", "Include: user asks to ignore policy", "State the boundary and route to an authorised person; no invented tool action", "Declared capability/permission boundary; deliberate challenge set" ] ], "note": "These are proposed cases, not a required sample size. Record “not applicable” with a reason for omitted cells. Keep representative sampling and deliberately enriched challenges separate when reporting results." }, "V27": { "title": "Recover from a lost write acknowledgement", "kind": "recovery", "article_url": "/blog/evaluate-ai-agents-tools-outcomes/", "section": "Test recovery deliberately", "source_sha256_before_addition": "4a1ca14b0d8be1e38194308ae742d8348efec4ff275c9ae581767a756c03dcad", "caption": "An illustrative action-recovery test. A confident final message is not proof that the intended state change happened.", "classification": "Illustrative action-recovery test · branches depend on authoritative state", "steps": [ [ "Operator request", "Declare the intended outcome and authorised action." ], [ "Agent and tool call", "Apply the tool’s permissions, budget and idempotency contract." ], [ "Response or timeout", "A missing acknowledgement does not prove that the write failed." ], [ "Read authoritative state", "Look up the operation or resulting state before a possible retry." ] ], "branches": [ [ "Confirmed success", "Report the verified state. Do not repeat the write." ], [ "Confirmed failure", "Follow the application’s documented recovery contract; retry only when that contract permits it." ], [ "Still unknown", "Retain the uncertainty and operation identity. Use documented recovery or escalate; do not issue a blind duplicate write." ] ], "note": "Arrows show the proposed test sequence, not measured causal effects. Score resulting state, action safety and truthful reporting, including incomplete or budget-exhausted runs." }, "V28": { "title": "Group related evidence before choosing a split", "kind": "split", "article_url": "/blog/evaluation-dataset-leakage-holdouts/", "section": "Split according to the generalisation you need", "source_sha256_before_addition": "935d33a980cc821bae7f6c081a08eb6da588aca63af9f8e80e7cf4e7fed19cfb", "caption": "Illustrative dataset boundaries. Related cases stay grouped; development and protected evaluation serve different purposes.", "classification": "Illustrative dataset boundary · no mandatory split percentages", "steps": [ [ "Source records", "Preserve the information available when each request was made." ], [ "Choose the independent unit", "Keep conversations, source/paraphrase variants, templates or repositories together as the question requires." ] ], "branches": [ [ "Development and selection", "Tune prompts/scorers and choose candidates with declared development and validation roles. Record what was exposed." ], [ "Protected final holdout", "Freeze the selected configuration before the declared final evaluation. Log exposure and follow the stated reuse/review policy." ] ], "note": "Assign whole groups, not related rows across both branches. A time split can test later work, but policy or labelling changes across the boundary must also be recorded. This diagram does not invent an automatic retirement rule." }, "V29": { "title": "What was shared, and what changed", "kind": "table", "article_url": "/blog/fair-ai-model-comparisons/", "section": "A real example of unmatched workflow conditions", "source_sha256_before_addition": "cc87ce0f2ed387dc785f2a9b50d4ee37735588d17dd8c02afe2a637ee80a50d4", "caption": "Recorded workflow conditions, represented in a later editorial table. These observations do not isolate a causal contribution from model weights.", "classification": "Recorded October 2026 workflow conditions · proposed controls kept separate", "headings": [ "Condition", "Recorded observation", "Possible future control — not a historical claim" ], "table": [ [ "Commission and evidence", "Shared brief, rubric and repository baseline; preserved original submissions", "Retain an exact versioned commission and evidence manifest" ], [ "Harness", "Sol/Luna: Codex CLI 0.163.0. DeepSeek: CommandCode CLI 1.88.0", "Use one harness when isolating a model substitution, or compare declared deployment configurations" ], [ "Tools, prompts and context", "Tool environment, system prompts and context handling differed", "Version these settings; match only what the intended comparison requires" ], [ "Provider and timing", "Provider and research date differed; DeepSeek workflow included command-approval waits", "Record provider/date and approval waits separately from inference measurements" ], [ "Effort", "“High” was a nominal setting, not evidence of matched computation", "Declare supported controls and observed budgets; do not infer equal compute" ], [ "Served model snapshot", "DeepSeek used the configured latest alias; a served-snapshot receipt was unavailable", "Retain a served-version receipt when the provider exposes one" ] ], "note": "These are conditions of a recorded workflow comparison. The score differences do not isolate the causal contribution of model weights, prompts, tools or provider. Unavailable settings remain unavailable." }, "V33": { "title": "Improve the suite without rewriting its history", "kind": "maintenance", "article_url": "/blog/maintain-llm-evaluation-suite/", "section": "Use layers with different jobs", "source_sha256_before_addition": "55029943f4054b0715dbc17e212f421b0d460a937663d09a933ed10e75d19439", "caption": "A proposed maintenance loop with versioned cases and decisions; adding data does not erase earlier results.", "classification": "Proposed maintenance loop · not a measured release process", "steps": [ [ "Observe a change or incident", "Retain the failing request, environment and observed outcome." ], [ "Triage the component", "Distinguish retrieval, model, tool, orchestration and policy failures." ], [ "Curate and adjudicate a case", "Specify expected behaviour and check the case’s independent unit." ], [ "Version development and regression coverage", "Retain old results; record the new case and its exposure." ], [ "Test the frozen release candidate", "Choose diagnostic, regression, representative and challenge layers for the change." ], [ "Record a decision and monitor", "Document residual risks and trigger a declared new version when needed." ] ], "branches": [ [ "Working layers", "Diagnostic cases, known-regression cases, representative samples and deliberate challenges answer different questions." ], [ "Protected holdout", "Keep access and reuse separately controlled for important new selection decisions. An exposed regression case is not fresh holdout evidence." ] ], "note": "The final monitoring step can start a newly versioned iteration. No universal cadence or minimum sample size is implied; adding cases never erases the earlier decision record." }, "V34": { "title": "An evidence card for a leaderboard result", "kind": "table", "article_url": "/blog/read-ai-benchmark-leaderboards/", "section": "Turn the reading into a shortlist", "source_sha256_before_addition": "e30e9202816ffc3ed768f21624ec866c711f86fe554a4bd2f9b579ebccd531b2", "caption": "A benchmark-reading template; a public score screens candidates rather than proving fitness for this workload.", "classification": "Reading template · no current ranking or vendor score asserted", "headings": [ "Field", "What to record", "Question before using the result" ], "table": [ [ "Task and dataset version", "Task definition, dataset/release version, source URL and access date", "Does it represent the work we need?" ], [ "Score and denominator", "What counts as success, scoring rule, case count and aggregation", "Which errors or slices could the total hide?" ], [ "Harness, tools and attempts", "Prompt/configuration, tools, retry/attempt budget and constraints", "Are these conditions comparable and deployable?" ], [ "Sample and coverage", "Independent unit, population, deliberate challenges and uncertainty", "What uncertainty or missing coverage remains?" ], [ "Grader", "Human/model judge, rubric, validation and known limitations", "Which mistakes can this grader miss?" ], [ "Exposure limits", "Dataset access, tuning or contamination disclosures", "Could this be familiarity rather than fresh generalisation?" ], [ "Local workload question", "Why shortlist this candidate; a test that could challenge that inference", "What bounded local evaluation should we run next?" ] ], "note": "Use a public score to screen candidates, then test the relevant local question. Confirm changing product specifications directly before committing to a choice." }, "V36": { "title": "Original round: judged quality and workflow elapsed time", "kind": "original-results", "article_url": "/blog/sol-vs-luna-editorial-benchmark-results/", "section": "The result at a glance", "source_sha256_before_addition": "de621b8a6d58f6a5a6db376edd0db7c4d110650defed9a5e80e580ad17434355", "caption": "Original 22 October 2026 round: judged quality and recorded workflow elapsed time measure different things; neither establishes general model superiority.", "classification": "Recorded original two-model round · 22 October 2026", "note": "One production run per model and one GPT-6.1 Sol model judge, sharing the winner’s designation. Workflow elapsed time includes tools and waiting; it is not inference speed or measured editing labour. Scores are rubric judgements, not probabilities. No statistical error bars or universal model ranking are supported. The later three-model review is a separate round." } }, "teaching_data": { "sol_metrics": { "classification": "Invented teaching data already present in article", "positive_class": "requires escalation", "n": 100, "tp": 15, "fp": 10, "fn": 5, "tn": 70, "precision": 0.6, "recall": 0.75, "f1": 0.6666666666666666, "accuracy": 0.85 }, "sol_judge": { "classification": "Invented teaching data already present in article", "positive_class": "human-adjudicated pass", "n": 100, "tp": 90, "fp": 10, "fn": 0, "tn": 0, "agreement": 0.9, "failure_detection_recall": 0 }, "sol_uncertainty": { "classification": "Calculated teaching example, not editorial model results", "successes": 80, "n": 100, "z": 1.96, "wilson_lower": 0.7111690380734977, "wilson_upper": 0.8666340774409013, "display_percent": [ 71.1, 80, 86.7 ], "paired_counts": { "both_pass": 75, "a_only": 5, "b_only": 10, "neither": 10 }, "a_successes": 80, "b_successes": 85 } }, "original_source_hashes": { "src/assets/evidence/ai-editorial-20261022/article-scores.json": "edbfa6b9db723fa53f70f4ad4b6c75dad7da5d4989b50005e8185825b50a36c6", "src/assets/evidence/ai-editorial-20261022/cluster-scores.json": "2bcbc44deee544e2d71d6c2728c59bd8bf978cd32777fa78de925a3aee3d18b8", "src/assets/evidence/ai-editorial-20261022/fact-check-ledger.json": "4b469e6cfc1c2a990e1a094756270dd7acefb2a5191bd60042258a6519d39c25", "src/assets/evidence/ai-editorial-20261022/frozen-rubric.json": "dccb6015970577c821a01750fa2353c75893b74be95a5d3f3a4fcea8835294cc", "src/assets/evidence/ai-editorial-20261022/original-source-sha256.json": "141b37cf8abc95b234f14d5b50e3a6182051393d24024aeea5379546c36bb218", "src/assets/evidence/ai-editorial-20261022/publication-edit-log.json": "6fd3c305d0ca1ac4c564664e45f7f2b41e00a707c3f5624b907f24eeadb34b0a", "src/assets/evidence/ai-editorial-20261022/token-usage.json": "4cb2d2ea426e1ac74dd21143575fa46aa027ecc98d3ca18dc9792e7a473934ac" }, "preparation_timezone": "Australia/Melbourne" }