{
  "performed_after_independent_score_freeze_utc": "2026-10-25T14:40:05.238744+00:00",
  "completed_at_utc": "2026-10-25T14:43:41.751705+00:00",
  "one_judge_only": true,
  "independent_evaluation_count": 1,
  "direction_count": 6,
  "question_mapping": [
    {
      "topic": "overall_workflow",
      "reader_question": "How do I turn a model choice into an evaluation and decision?",
      "articles": {
        "A": "A/articles/choosing-model-benchmark.md",
        "B": "B/articles/ai-model-benchmarking-guide.md",
        "C": "C/articles/evaluating-ai-models-and-agent-systems.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "public_evidence",
      "reader_question": "What can a public score tell me, and what should I test locally?",
      "articles": {
        "A": "A/articles/public-benchmark-literacy.md",
        "B": "B/articles/read-ai-benchmark-leaderboards.md",
        "C": "C/articles/public-benchmarks-vs-workload-evaluation.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "task_design",
      "reader_question": "How do I specify cases, expected behaviour and coverage?",
      "articles": {
        "A": "A/articles/build-task-eval.md",
        "B": "B/articles/design-llm-evaluation-tasks.md",
        "C": "C/articles/designing-evaluation-tasks-and-datasets.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "data_integrity",
      "reader_question": "How do I protect splits and investigate contamination?",
      "articles": {
        "A": "A/articles/datasets-holdouts-contamination.md",
        "B": "B/articles/evaluation-dataset-leakage-holdouts.md",
        "C": "C/articles/contamination-leakage-and-holdouts.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "metrics",
      "reader_question": "How should I calculate scores and apply a rubric?",
      "articles": {
        "A": "A/articles/metrics-rubrics.md",
        "B": "B/articles/llm-evaluation-metrics-rubrics.md",
        "C": "C/articles/objective-metrics-and-rubrics.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "judges",
      "reader_question": "How should humans and model judges grade outputs?",
      "articles": {
        "A": "A/articles/human-evaluation-model-judges.md",
        "B": "B/articles/human-reviewers-llm-judges.md",
        "C": "C/articles/human-evaluation-and-model-judges.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "uncertainty",
      "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
      "articles": {
        "A": "A/articles/repeatability-uncertainty.md",
        "B": "B/articles/benchmark-uncertainty-repeatability.md",
        "C": "C/articles/repeatability-variance-and-uncertainty.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "fairness",
      "reader_question": "Which prompts, tools and configuration choices must I control?",
      "articles": {
        "A": "A/articles/fair-model-comparisons.md",
        "B": "B/articles/fair-ai-model-comparisons.md",
        "C": "C/articles/prompt-tool-and-configuration-fairness.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "economics",
      "reader_question": "What does accepted work cost, and how long does it take?",
      "articles": {
        "A": "A/articles/latency-cost.md",
        "B": "B/articles/ai-latency-cost-per-successful-task.md",
        "C": "C/articles/latency-token-usage-and-cost.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "agents",
      "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
      "articles": {
        "A": "A/articles/agent-system-evaluation.md",
        "B": "B/articles/evaluate-ai-agents-tools-outcomes.md",
        "C": "C/articles/evaluating-agents-and-tool-use.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    },
    {
      "topic": "maintenance",
      "reader_question": "How do I diagnose changes and maintain decision evidence?",
      "articles": {
        "A": "A/articles/maintaining-evaluations.md",
        "B": "B/articles/maintain-llm-evaluation-suite.md",
        "C": "C/articles/interpreting-results-and-maintaining-evals.md"
      },
      "matching_basis": "Overlapping reader decision and substantive full-draft treatment, not exact title equality."
    }
  ],
  "passes": [
    {
      "direction": "A→B",
      "presentation_order": [
        "A",
        "B"
      ],
      "pass_kind": "first_order",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "A": "A/articles/choosing-model-benchmark.md",
            "B": "B/articles/ai-model-benchmarking-guide.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A supplies the fuller invoice narrative; B’s six concrete artefacts and bounded first-evaluation plan make the overall workflow easier to start."
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "A": "A/articles/public-benchmark-literacy.md",
            "B": "B/articles/read-ai-benchmark-leaderboards.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s checklist is good; B’s evidence card, HumanEval attempt distinction and correctly dated common-harness example are better supported."
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "A": "A/articles/build-task-eval.md",
            "B": "B/articles/design-llm-evaluation-tasks.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "A has a fuller label/sampling protocol; B has a more precise next-turn observation point and RETURN-014 record. These are offsetting practical strengths."
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "A": "A/articles/datasets-holdouts-contamination.md",
            "B": "B/articles/evaluation-dataset-leakage-holdouts.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s custody/versioning guidance is sound; B more cleanly separates leakage, contamination and mismatch with directly relevant evidence."
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "A": "A/articles/metrics-rubrics.md",
            "B": "B/articles/llm-evaluation-metrics-rubrics.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A explains metric choices well; B works the full confusion matrix and connects a 0/1/2 rubric to a success conjunction."
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "A": "A/articles/human-evaluation-model-judges.md",
            "B": "B/articles/human-reviewers-llm-judges.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A offers a useful hybrid workflow; B’s calibration/validation split, false-pass example and adversarial judge checks are more concrete."
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "A": "A/articles/repeatability-uncertainty.md",
            "B": "B/articles/benchmark-uncertainty-repeatability.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A identifies dependence correctly but leaves interval construction open; B provides a Wilson calculation and paired-bootstrap recipe with caveats."
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "A": "A/articles/fair-model-comparisons.md",
            "B": "B/articles/fair-ai-model-comparisons.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Both distinguish controlled and best-configuration comparisons. B adds primary prompt-format evidence, prior incumbent investment and unsupported-control treatment."
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "A": "A/articles/latency-cost.md",
            "B": "B/articles/ai-latency-cost-per-successful-task.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A includes human-review economics; B’s disjoint usage categories, zero-success handling and censored latency avoid more accounting mistakes."
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "A": "A/articles/agent-system-evaluation.md",
            "B": "B/articles/evaluate-ai-agents-tools-outcomes.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A has a good coding sandbox and ablation example. B’s lost-write acknowledgement, permission predicates and simulator limits add the more specific recovery protocol."
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "A": "A/articles/maintaining-evaluations.md",
            "B": "B/articles/maintain-llm-evaluation-suite.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Both version and refresh data. B gives a sharper input/output/scorer regression triage and a decision-expiry trigger."
        }
      ],
      "overall_preference": "B",
      "basis": "Overall recommendation uses frozen weights and publication holds, not a vote count over topics."
    },
    {
      "direction": "B→A",
      "presentation_order": [
        "B",
        "A"
      ],
      "pass_kind": "reverse_order_internal_bias_check",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "A": "A/articles/choosing-model-benchmark.md",
            "B": "B/articles/ai-model-benchmarking-guide.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Starting with B’s concise workflow, A adds useful invoice risk gates but no missing procedural step sufficient to reverse the preference.",
          "changed_from_forward": false
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "A": "A/articles/public-benchmark-literacy.md",
            "B": "B/articles/read-ai-benchmark-leaderboards.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "B’s benchmark audit survives comparison with A’s broader checklist; A’s wrong Shi author label remains a small but real repair.",
          "changed_from_forward": false
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "A": "A/articles/build-task-eval.md",
            "B": "B/articles/design-llm-evaluation-tasks.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "Beginning with B’s concrete case does not erase A’s useful 120-case sampling/labelling walkthrough. Neither support clearly dominates this question.",
          "changed_from_forward": false
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "A": "A/articles/datasets-holdouts-contamination.md",
            "B": "B/articles/evaluation-dataset-leakage-holdouts.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A has a recent, well-qualified preprint, but B’s context-assembly audit and careful exposure limits remain the more exact implementation guide.",
          "changed_from_forward": false
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "A": "A/articles/metrics-rubrics.md",
            "B": "B/articles/llm-evaluation-metrics-rubrics.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "B’s executable arithmetic remains the deciding asset after reading A’s broad metric taxonomy and scorecard.",
          "changed_from_forward": false
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "A": "A/articles/human-evaluation-model-judges.md",
            "B": "B/articles/human-reviewers-llm-judges.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s disagreement guidance is valuable, but B also handles unscorable evidence and does not overstate family independence.",
          "changed_from_forward": false
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "A": "A/articles/repeatability-uncertainty.md",
            "B": "B/articles/benchmark-uncertainty-repeatability.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "The advantage persists when B is read first: A supplies no worked interval to offset B’s greater followability.",
          "changed_from_forward": false
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "A": "A/articles/fair-model-comparisons.md",
            "B": "B/articles/fair-ai-model-comparisons.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s clear incident-summary example helps, but B’s explicit lack of internal-compute equivalence is more useful across providers.",
          "changed_from_forward": false
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "A": "A/articles/latency-cost.md",
            "B": "B/articles/ai-latency-cost-per-successful-task.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s $258/$127 example is an advantage for labour economics, but its unfinished success-denominator worksheet does not overcome B’s precise accounting.",
          "changed_from_forward": false
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "A": "A/articles/agent-system-evaluation.md",
            "B": "B/articles/evaluate-ai-agents-tools-outcomes.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A covers the broad system boundary well; B still better answers what to do after an ambiguous state-changing operation.",
          "changed_from_forward": false
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "A": "A/articles/maintaining-evaluations.md",
            "B": "B/articles/maintain-llm-evaluation-suite.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s anchor/rolling bridge is useful; B’s policy-version example and ownership remain the more directly diagnostic operating guide.",
          "changed_from_forward": false
        }
      ],
      "overall_preference": "B",
      "reversal_changed_overall_judgement": false,
      "reversal_changed_topic_judgements": [],
      "interpretation": "No judgment changed in this deliberate reverse-order reconsideration. The judge remembers the submissions; this is not an independent replication or proof that order bias is absent."
    },
    {
      "direction": "A→C",
      "presentation_order": [
        "A",
        "C"
      ],
      "pass_kind": "first_order",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "A": "A/articles/choosing-model-benchmark.md",
            "C": "C/articles/evaluating-ai-models-and-agent-systems.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C is more engaging and research-dense; A’s explicit hard gates and conditional sampling advice create a safer first decision workflow."
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "A": "A/articles/public-benchmark-literacy.md",
            "C": "C/articles/public-benchmarks-vs-workload-evaluation.md"
          },
          "preferred_cluster": "C",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s verified research breadth and shortlist scorecard narrowly outweigh A’s concise checklist and small citation repair for this external-evidence question."
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "A": "A/articles/build-task-eval.md",
            "C": "C/articles/designing-evaluation-tasks-and-datasets.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C supplies a copyable YAML schema; A better handles grouped splits, legitimate ambiguity and differing sampling goals without blanket synthetic-data bans."
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "A": "A/articles/datasets-holdouts-contamination.md",
            "C": "C/articles/contamination-leakage-and-holdouts.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s contamination study numbers are mostly right; A avoids C’s unsupported upper-bound rule and conflicting retirement policy."
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "A": "A/articles/metrics-rubrics.md",
            "C": "C/articles/objective-metrics-and-rubrics.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C has more metric families and pass@k notation; A’s normalisation and class-balance qualifications avoid C’s broad F1/verbosity claim."
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "A": "A/articles/human-evaluation-model-judges.md",
            "C": "C/articles/human-evaluation-and-model-judges.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A’s wrong author label is a small repair. C needs broader qualification of self-enhancement certainty, averaging and comparative grading claims."
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "A": "A/articles/repeatability-uncertainty.md",
            "C": "C/articles/repeatability-variance-and-uncertainty.md"
          },
          "preferred_cluster": "C",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s checked SE, paired-difference and power examples provide a followable calculation that A lacks; approximation caveats still need editing."
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "A": "A/articles/fair-model-comparisons.md",
            "C": "C/articles/prompt-tool-and-configuration-fairness.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C has a useful manifest; A more clearly limits equivalent settings to valid common controls and avoids asserting that effort always increases work."
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "A": "A/articles/latency-cost.md",
            "C": "C/articles/latency-token-usage-and-cost.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C adds correct dated rates and marginal economics, but its visible-output billing definition and missing Haiku price bracket require substantive repair. A has a smaller bookkeeping clarification."
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "A": "A/articles/agent-system-evaluation.md",
            "C": "C/articles/evaluating-agents-and-tool-use.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "A offers a concrete repository sandbox and a recent leakage caution; C offers a fuller failure taxonomy and verified benchmark context. Both need more concrete recovery fixtures."
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "A": "A/articles/maintaining-evaluations.md",
            "C": "C/articles/interpreting-results-and-maintaining-evals.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s memo template and live deprecation example are useful; A better preserves stable anchors and avoids blanket retirement of saturated gate tests."
        }
      ],
      "overall_preference": "A",
      "basis": "Overall recommendation uses frozen weights and publication holds, not a vote count over topics."
    },
    {
      "direction": "C→A",
      "presentation_order": [
        "C",
        "A"
      ],
      "pass_kind": "reverse_order_internal_bias_check",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "A": "A/articles/choosing-model-benchmark.md",
            "C": "C/articles/evaluating-ai-models-and-agent-systems.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Starting with C’s six-step loop highlights useful formulas, but A avoids turning small pilots into non-evaluations or prescribing a generic minimum.",
          "changed_from_forward": false
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "A": "A/articles/public-benchmark-literacy.md",
            "C": "C/articles/public-benchmarks-vs-workload-evaluation.md"
          },
          "preferred_cluster": "C",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Reading C first, A is simpler and less survey-like, but C’s source-backed distortion/coverage examples and table retain a narrow advantage.",
          "changed_from_forward": false
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "A": "A/articles/build-task-eval.md",
            "C": "C/articles/designing-evaluation-tasks-and-datasets.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s schema remains a real asset; A’s careful sampling/labelling narrative carries the operational decision more reliably.",
          "changed_from_forward": false
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "A": "A/articles/datasets-holdouts-contamination.md",
            "C": "C/articles/contamination-leakage-and-holdouts.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s stronger research catalogue does not justify its deployment-performance upper bound; A’s unknown-exposure conclusion is more defensible.",
          "changed_from_forward": false
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "A": "A/articles/metrics-rubrics.md",
            "C": "C/articles/objective-metrics-and-rubrics.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "A remains preferable after starting with C’s rubric table: trustworthy interpretation matters more than adding metric names.",
          "changed_from_forward": false
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "A": "A/articles/human-evaluation-model-judges.md",
            "C": "C/articles/human-evaluation-and-model-judges.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s kappa formula and validation steps add depth, but A’s process treats order reversal as a diagnostic and has fewer unsupported guarantees.",
          "changed_from_forward": false
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "A": "A/articles/repeatability-uncertainty.md",
            "C": "C/articles/repeatability-variance-and-uncertainty.md"
          },
          "preferred_cluster": "C",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Starting with C makes its caveat gaps visible, but A’s high-level discussion cannot replace C’s concrete statistical toolkit.",
          "changed_from_forward": false
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "A": "A/articles/fair-model-comparisons.md",
            "C": "C/articles/prompt-tool-and-configuration-fairness.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "The copyable C manifest is attractive, yet A’s qualification of API asymmetry better serves a cross-provider experiment.",
          "changed_from_forward": false
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "A": "A/articles/latency-cost.md",
            "C": "C/articles/latency-token-usage-and-cost.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s richer table does not remove its undercount risk. A’s simpler example is more trustworthy as submitted, though it needs a completed denominator.",
          "changed_from_forward": false
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "A": "A/articles/agent-system-evaluation.md",
            "C": "C/articles/evaluating-agents-and-tool-use.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "C’s system decomposition and A’s ablation example answer complementary needs. Neither clearly dominates this support when separated from their cost articles.",
          "changed_from_forward": false
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "A": "A/articles/maintaining-evaluations.md",
            "C": "C/articles/interpreting-results-and-maintaining-evals.md"
          },
          "preferred_cluster": "A",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s operating framework is strong, but A’s bridge/version discussion is more internally consistent with its data article.",
          "changed_from_forward": false
        }
      ],
      "overall_preference": "A",
      "reversal_changed_overall_judgement": false,
      "reversal_changed_topic_judgements": [],
      "interpretation": "No judgment changed in this deliberate reverse-order reconsideration. The judge remembers the submissions; this is not an independent replication or proof that order bias is absent."
    },
    {
      "direction": "B→C",
      "presentation_order": [
        "B",
        "C"
      ],
      "pass_kind": "first_order",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "B": "B/articles/ai-model-benchmarking-guide.md",
            "C": "C/articles/evaluating-ai-models-and-agent-systems.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C offers richer research context and a more dramatic opening. B’s bounded artefact checklist and coherent support-assistant example better guide a first evaluation."
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "B": "B/articles/read-ai-benchmark-leaderboards.md",
            "C": "C/articles/public-benchmarks-vs-workload-evaluation.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "B’s evidence-card audit and C’s verified shortlist scorecard/research examples are both strong; precision versus breadth is a reader preference here."
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "B": "B/articles/design-llm-evaluation-tasks.md",
            "C": "C/articles/designing-evaluation-tasks-and-datasets.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s schema is copyable; B’s observation point, contrastive/invariance cases and explicit unscorable distinction better preserve task validity."
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "B": "B/articles/evaluation-dataset-leakage-holdouts.md",
            "C": "C/articles/contamination-leakage-and-holdouts.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C has more contamination numbers; B states method assumptions, training-exposure limits and split roles without the erroneous upper-bound inference."
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "B": "B/articles/llm-evaluation-metrics-rubrics.md",
            "C": "C/articles/objective-metrics-and-rubrics.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s taxonomy is broader. B’s complete confusion matrix, anchored policy score and explicit denominator treatment give clearer implementation."
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "B": "B/articles/human-reviewers-llm-judges.md",
            "C": "C/articles/human-evaluation-and-model-judges.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Both require calibration. B more carefully distinguishes validator evidence, false passes, order flips and injection controls; C overstates some mechanisms."
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "B": "B/articles/benchmark-uncertainty-repeatability.md",
            "C": "C/articles/repeatability-variance-and-uncertainty.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C has more formulas and power detail; B’s calculated Wilson interval and paired resampling are more tightly bounded for sparse/related cases."
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "B": "B/articles/fair-ai-model-comparisons.md",
            "C": "C/articles/prompt-tool-and-configuration-fairness.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C supplies a manifest. B adds prompt-format evidence and accurately warns that nominal settings do not establish internal-compute matching."
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "B": "B/articles/ai-latency-cost-per-successful-task.md",
            "C": "C/articles/latency-token-usage-and-cost.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s rates and marginal-cost illustration add context; B defines disjoint billing categories and accepted-task ratios without C’s reasoning-token ambiguity."
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "B": "B/articles/evaluate-ai-agents-tools-outcomes.md",
            "C": "C/articles/evaluating-agents-and-tool-use.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s taxonomy and historical context are stronger survey assets; B’s state/permission predicates, ambiguous-write recovery and simulator audit better answer the operational question."
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "B": "B/articles/maintain-llm-evaluation-suite.md",
            "C": "C/articles/interpreting-results-and-maintaining-evals.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C has a copyable memo and current platform notice. B provides more exact regression triage, incident sampling limits and immutable-label repair."
        }
      ],
      "overall_preference": "B",
      "basis": "Overall recommendation uses frozen weights and publication holds, not a vote count over topics."
    },
    {
      "direction": "C→B",
      "presentation_order": [
        "C",
        "B"
      ],
      "pass_kind": "reverse_order_internal_bias_check",
      "topic_judgements": [
        {
          "topic": "overall_workflow",
          "reader_question": "How do I turn a model choice into an evaluation and decision?",
          "evidence_files": {
            "B": "B/articles/ai-model-benchmarking-guide.md",
            "C": "C/articles/evaluating-ai-models-and-agent-systems.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s greater range does not outweigh B’s clearer decision contract and fewer categorical sample-size rules.",
          "changed_from_forward": false
        },
        {
          "topic": "public_evidence",
          "reader_question": "What can a public score tell me, and what should I test locally?",
          "evidence_files": {
            "B": "B/articles/read-ai-benchmark-leaderboards.md",
            "C": "C/articles/public-benchmarks-vs-workload-evaluation.md"
          },
          "preferred_cluster": "tie",
          "uncertainty": "Editorial tie/close question.",
          "rationale": "Starting with C’s research survey makes B’s concision attractive, but not decisively superior for this reader question. The tie remains.",
          "changed_from_forward": false
        },
        {
          "topic": "task_design",
          "reader_question": "How do I specify cases, expected behaviour and coverage?",
          "evidence_files": {
            "B": "B/articles/design-llm-evaluation-tasks.md",
            "C": "C/articles/designing-evaluation-tasks-and-datasets.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "The C schema does not compensate for its unsupported synthetic-data prohibition and absence of B’s clear time-of-request boundary.",
          "changed_from_forward": false
        },
        {
          "topic": "data_integrity",
          "reader_question": "How do I protect splits and investigate contamination?",
          "evidence_files": {
            "B": "B/articles/evaluation-dataset-leakage-holdouts.md",
            "C": "C/articles/contamination-leakage-and-holdouts.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s benchmark history is useful, but B better separates an overlap audit from proof that data are unseen.",
          "changed_from_forward": false
        },
        {
          "topic": "metrics",
          "reader_question": "How should I calculate scores and apply a rubric?",
          "evidence_files": {
            "B": "B/articles/llm-evaluation-metrics-rubrics.md",
            "C": "C/articles/objective-metrics-and-rubrics.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "The extra C formulas do not reverse B’s advantage in worked arithmetic and metric-specific limitations.",
          "changed_from_forward": false
        },
        {
          "topic": "judges",
          "reader_question": "How should humans and model judges grade outputs?",
          "evidence_files": {
            "B": "B/articles/human-reviewers-llm-judges.md",
            "C": "C/articles/human-evaluation-and-model-judges.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s agreement-statistics discussion adds vocabulary; B’s concrete audit remains the stronger guide for trusting automated acceptance.",
          "changed_from_forward": false
        },
        {
          "topic": "uncertainty",
          "reader_question": "How do I interpret repeats, paired differences and uncertainty?",
          "evidence_files": {
            "B": "B/articles/benchmark-uncertainty-repeatability.md",
            "C": "C/articles/repeatability-variance-and-uncertainty.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s verified power arithmetic is valuable, but B’s procedural completeness and explicit non-inferiority/stopping limits retain the preference.",
          "changed_from_forward": false
        },
        {
          "topic": "fairness",
          "reader_question": "Which prompts, tools and configuration choices must I control?",
          "evidence_files": {
            "B": "B/articles/fair-ai-model-comparisons.md",
            "C": "C/articles/prompt-tool-and-configuration-fairness.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Starting with C’s template exposes its portability assumptions; B more clearly records unsupported controls and actual usage.",
          "changed_from_forward": false
        },
        {
          "topic": "economics",
          "reader_question": "What does accepted work cost, and how long does it take?",
          "evidence_files": {
            "B": "B/articles/ai-latency-cost-per-successful-task.md",
            "C": "C/articles/latency-token-usage-and-cost.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "Correct C headline rates cannot offset a material long-context omission and an undercounting formula; B remains preferable.",
          "changed_from_forward": false
        },
        {
          "topic": "agents",
          "reader_question": "How do I evaluate tools, state changes, permissions and recovery?",
          "evidence_files": {
            "B": "B/articles/evaluate-ai-agents-tools-outcomes.md",
            "C": "C/articles/evaluating-agents-and-tool-use.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s reset/ablation controls are sound; B still provides the more concrete failure-injection and recovery method.",
          "changed_from_forward": false
        },
        {
          "topic": "maintenance",
          "reader_question": "How do I diagnose changes and maintain decision evidence?",
          "evidence_files": {
            "B": "B/articles/maintain-llm-evaluation-suite.md",
            "C": "C/articles/interpreting-results-and-maintaining-evals.md"
          },
          "preferred_cluster": "B",
          "uncertainty": "Submission-level editorial preference, not a statistical effect.",
          "rationale": "C’s retirement rules need reconciliation; B’s distinction between changed input, changed output and changed scorer remains more useful.",
          "changed_from_forward": false
        }
      ],
      "overall_preference": "B",
      "reversal_changed_overall_judgement": false,
      "reversal_changed_topic_judgements": [],
      "interpretation": "No judgment changed in this deliberate reverse-order reconsideration. The judge remembers the submissions; this is not an independent replication or proof that order bias is absent."
    }
  ],
  "ties_allowed": true,
  "score_files_changed_by_comparison": false,
  "limits": "Six directions are three ordered comparisons plus their internal reversals by the same judge. Eleven overlapping questions are editorial lenses, not independent model runs or statistical samples. No new weights or target winner imposed."
}