{ "schema_version": 7, "benchmark": "invoice-approval-sop-to-validated-draft", "generated_at": "2026-08-02T22:35:58.342033+00:00", "method": { "surface": "Sanctum Chat public API", "persona": "surgeon", "fixed_model": "opencode-go/qwen3.7-max", "max_output_tokens": 32768, "write_confirmation": "operator-approved; never automatic", "direct_flow_api_calls": false, "manual_bpmn_xml_edits": false, "document_extraction_required": true, "acceptance": "retrieved definition semantics plus Flow lint, 25-gate quality, grammar, and deterministic non-persisted structural preview results", "subprocess_outcome_correlation_modeled": false, "preview_is_decision_grade_simulation": false }, "runs_requested": 3, "runs_attempted": 3, "runs_completed": 3, "runs_failed": 0, "elapsed_seconds": { "median": 1228.152, "minimum": 852.955, "maximum": 1317.586 }, "runs": [ { "run": 1, "started_at": "2026-08-02T21:39:19.634473+00:00", "finished_at": "2026-08-02T21:59:47.788766+00:00", "completed_at": "2026-08-02T21:59:47.788766+00:00", "elapsed_seconds": 1228.152, "completion_marker_seen": true, "completion_signal_valid": true, "status": "completed", "completed": true, "failure_code": null, "acceptance_checks": { "completion_marker_present": true, "completion_signal_valid": true, "completion_prose_consistent": true, "definition_key_reported": true, "complete_source_read": true, "draft_create_attempted": true, "draft_create_confirmation_blocked": true, "precreate_semantic_validation_passed": true, "operator_approved_write": true, "definition_update_not_attempted": true, "pending_write_confirmed": true, "definition_retrieved": true, "lint_completed": true, "quality_audit_completed": true, "grammar_audit_completed": true, "simulation_preview_completed": true, "required_read_tools_completed": true, "definition_not_published": true, "zero_factual_clarifications": true, "fixed_model_used": true, "independent_semantic_validation_passed": true }, "turn_metrics": [ { "tool_call_count": 9, "completed_tool_count": 3, "blocked_write_count": 1, "failed_tool_count": 5, "controlled_execution_completed": false, "controlled_execution_declined": false, "precreate_semantic_validation_present": true, "precreate_semantic_validation_passed": true, "precreate_semantic_failed_check_count": 0 }, { "tool_call_count": 6, "completed_tool_count": 6, "blocked_write_count": 0, "failed_tool_count": 0, "controlled_execution_completed": true, "controlled_execution_declined": false, "precreate_semantic_validation_present": false, "precreate_semantic_validation_passed": false, "precreate_semantic_failed_check_count": -1 } ], "clarification_turns": 0, "write_approvals": 1, "tool_call_count": 15, "semantic_validation": { "passed": true, "checks": { "evidence_order_unambiguous": true, "source_attachment_evidence_present": true, "source_attachment_identity_matches": true, "benchmark_source_matches_manifest": true, "source_attachment_extraction_complete": true, "source_attachment_complete_read": true, "source_read_before_definition_write": true, "only_attachment_source_used": true, "all_definition_reads_bound_to_created_definition": true, "controlled_write_chain_evidence_present": true, "create_confirmation_chain_valid": true, "exactly_one_approved_definition_create": true, "no_non_create_definition_mutation_attempted": true, "all_writes_explicitly_confirmed": true, "definition_write_scope_valid": true, "manual_bpmn_xml_not_supplied": true, "validation_evidence_after_last_write": true, "validation_chain_bound_to_final_definition": true, "simulation_preview_result_matches_definition": true, "final_get_after_validations": true, "no_later_definition_mutation": true, "definition_never_published": true, "preparation_subprocess_semantics_exact": true, "semantic_role_mapping_complete": true, "semantic_graph_exact": true, "global_bpmn_element_ids_unique": true, "top_level_routing_probabilities_exact": true, "approval_routing_probabilities_exact": true, "timing_parameters_exact": true, "sla_parameters_exact": true, "required_data_io_exact": true, "data_association_inventory_exact": true, "review_approval_control_semantics_exact": true, "boundary_event_inventory_exact": true, "boundary_event_configuration_exact": true, "multi_instance_inventory_exact": true, "multi_instance_configuration_exact": true, "timing_inventory_exact": true, "sla_inventory_exact": true, "gateway_branch_polarity_exact": true, "event_reference_inventory_exact": true, "non_gateway_flow_semantics_exact": true, "implementation_inventory_exact": true, "activity_descriptions_exact": true, "supplier_crossings_exact": true, "governance_metadata_complete": true, "semantic_contradictions_absent": true, "definition_evidence_present": true, "definition_key_matches": true, "definition_name_matches": true, "definition_description_absent": true, "definition_category_absent": true, "definition_remains_draft": true, "definition_non_executable": true, "definition_uses_simulation_profile": true, "source_pool_identity_exact": true, "participant_minimum_met": true, "external_supplier_present": true, "participant_inventory_exact": true, "participant_accountability_exact": true, "assignee_accountability_exact": true, "message_flow_minimum_met": true, "supplier_message_flows_complete": true, "message_flow_semantics_exact": true, "data_object_minimum_met": true, "required_data_records_present": true, "data_object_inventory_exact": true, "data_object_descriptions_absent": true, "accountable_lane_minimum_met": true, "required_accountable_lanes_present": true, "substantial_graph_present": true, "required_bpmn_elements_present": true, "message_start_and_end_outcomes_present": true, "terminal_outcomes_exact": true, "interrupting_pt48h_escalation_present": true, "sequential_multi_instance_approval_present": true, "scope_exclusions_documented": true, "forbidden_payment_work_absent": true, "excluded_implementation_absent": true, "excluded_scope_contradictions_absent": true, "gateway_probabilities_valid": true, "gateway_defaults_valid": true, "quality_audit_evidence_present": true, "quality_audit_all_gates_passed": true, "quality_audit_includes_gate_14_suite": true, "grammar_audit_evidence_present": true, "grammar_audit_passed": true, "grammar_warnings_clear": true, "lint_evidence_present": true, "lint_passed": true, "lint_simulation_ready": true, "simulation_preview_evidence_present": true, "simulation_preview_non_persisted": true, "simulation_preview_draft_bound": true, "simulation_preview_deterministic_config": true, "simulation_preview_limitations_disclosed": true, "simulation_preview_metrics_finite": true, "simulation_preview_step_coverage_complete": true }, "metrics": { "source_read_page_count": 2, "source_line_count": 139, "write_proposal_count": 1, "write_resolution_count": 1, "approved_write_count": 1, "successful_definition_write_count": 1, "successful_definition_update_count": 0, "bound_validation_count": 4, "participant_count": 2, "message_flow_count": 4, "supplier_message_flow_count": 4, "data_object_count": 6, "accountable_lane_count": 4, "top_level_step_count": 18, "total_step_count": 32, "terminal_outcome_count": 4, "exclusive_gateway_count": 7, "converging_gateway_count": 3, "quality_passed_gates": 25, "quality_failed_gates": 0, "quality_total_gates": 25, "grammar_error_count": 0, "grammar_warning_count": 0, "lint_error_count": 0, "lint_warning_count": 3, "simulation_preview_run_count": 1000, "simulation_preview_step_count": 31, "failed_check_count": 0 } } }, { "run": 2, "started_at": "2026-08-02T21:59:47.789958+00:00", "finished_at": "2026-08-02T22:21:45.376740+00:00", "completed_at": "2026-08-02T22:21:45.376740+00:00", "elapsed_seconds": 1317.586, "completion_marker_seen": true, "completion_signal_valid": true, "status": "completed", "completed": true, "failure_code": null, "acceptance_checks": { "completion_marker_present": true, "completion_signal_valid": true, "completion_prose_consistent": true, "definition_key_reported": true, "complete_source_read": true, "draft_create_attempted": true, "draft_create_confirmation_blocked": true, "precreate_semantic_validation_passed": true, "operator_approved_write": true, "definition_update_not_attempted": true, "pending_write_confirmed": true, "definition_retrieved": true, "lint_completed": true, "quality_audit_completed": true, "grammar_audit_completed": true, "simulation_preview_completed": true, "required_read_tools_completed": true, "definition_not_published": true, "zero_factual_clarifications": true, "fixed_model_used": true, "independent_semantic_validation_passed": true }, "turn_metrics": [ { "tool_call_count": 12, "completed_tool_count": 3, "blocked_write_count": 1, "failed_tool_count": 8, "controlled_execution_completed": false, "controlled_execution_declined": false, "precreate_semantic_validation_present": true, "precreate_semantic_validation_passed": true, "precreate_semantic_failed_check_count": 0 }, { "tool_call_count": 6, "completed_tool_count": 6, "blocked_write_count": 0, "failed_tool_count": 0, "controlled_execution_completed": true, "controlled_execution_declined": false, "precreate_semantic_validation_present": false, "precreate_semantic_validation_passed": false, "precreate_semantic_failed_check_count": -1 } ], "clarification_turns": 0, "write_approvals": 1, "tool_call_count": 18, "semantic_validation": { "passed": true, "checks": { "evidence_order_unambiguous": true, "source_attachment_evidence_present": true, "source_attachment_identity_matches": true, "benchmark_source_matches_manifest": true, "source_attachment_extraction_complete": true, "source_attachment_complete_read": true, "source_read_before_definition_write": true, "only_attachment_source_used": true, "all_definition_reads_bound_to_created_definition": true, "controlled_write_chain_evidence_present": true, "create_confirmation_chain_valid": true, "exactly_one_approved_definition_create": true, "no_non_create_definition_mutation_attempted": true, "all_writes_explicitly_confirmed": true, "definition_write_scope_valid": true, "manual_bpmn_xml_not_supplied": true, "validation_evidence_after_last_write": true, "validation_chain_bound_to_final_definition": true, "simulation_preview_result_matches_definition": true, "final_get_after_validations": true, "no_later_definition_mutation": true, "definition_never_published": true, "preparation_subprocess_semantics_exact": true, "semantic_role_mapping_complete": true, "semantic_graph_exact": true, "global_bpmn_element_ids_unique": true, "top_level_routing_probabilities_exact": true, "approval_routing_probabilities_exact": true, "timing_parameters_exact": true, "sla_parameters_exact": true, "required_data_io_exact": true, "data_association_inventory_exact": true, "review_approval_control_semantics_exact": true, "boundary_event_inventory_exact": true, "boundary_event_configuration_exact": true, "multi_instance_inventory_exact": true, "multi_instance_configuration_exact": true, "timing_inventory_exact": true, "sla_inventory_exact": true, "gateway_branch_polarity_exact": true, "event_reference_inventory_exact": true, "non_gateway_flow_semantics_exact": true, "implementation_inventory_exact": true, "activity_descriptions_exact": true, "supplier_crossings_exact": true, "governance_metadata_complete": true, "semantic_contradictions_absent": true, "definition_evidence_present": true, "definition_key_matches": true, "definition_name_matches": true, "definition_description_absent": true, "definition_category_absent": true, "definition_remains_draft": true, "definition_non_executable": true, "definition_uses_simulation_profile": true, "source_pool_identity_exact": true, "participant_minimum_met": true, "external_supplier_present": true, "participant_inventory_exact": true, "participant_accountability_exact": true, "assignee_accountability_exact": true, "message_flow_minimum_met": true, "supplier_message_flows_complete": true, "message_flow_semantics_exact": true, "data_object_minimum_met": true, "required_data_records_present": true, "data_object_inventory_exact": true, "data_object_descriptions_absent": true, "accountable_lane_minimum_met": true, "required_accountable_lanes_present": true, "substantial_graph_present": true, "required_bpmn_elements_present": true, "message_start_and_end_outcomes_present": true, "terminal_outcomes_exact": true, "interrupting_pt48h_escalation_present": true, "sequential_multi_instance_approval_present": true, "scope_exclusions_documented": true, "forbidden_payment_work_absent": true, "excluded_implementation_absent": true, "excluded_scope_contradictions_absent": true, "gateway_probabilities_valid": true, "gateway_defaults_valid": true, "quality_audit_evidence_present": true, "quality_audit_all_gates_passed": true, "quality_audit_includes_gate_14_suite": true, "grammar_audit_evidence_present": true, "grammar_audit_passed": true, "grammar_warnings_clear": true, "lint_evidence_present": true, "lint_passed": true, "lint_simulation_ready": true, "simulation_preview_evidence_present": true, "simulation_preview_non_persisted": true, "simulation_preview_draft_bound": true, "simulation_preview_deterministic_config": true, "simulation_preview_limitations_disclosed": true, "simulation_preview_metrics_finite": true, "simulation_preview_step_coverage_complete": true }, "metrics": { "source_read_page_count": 2, "source_line_count": 139, "write_proposal_count": 1, "write_resolution_count": 1, "approved_write_count": 1, "successful_definition_write_count": 1, "successful_definition_update_count": 0, "bound_validation_count": 4, "participant_count": 2, "message_flow_count": 4, "supplier_message_flow_count": 4, "data_object_count": 6, "accountable_lane_count": 4, "top_level_step_count": 18, "total_step_count": 32, "terminal_outcome_count": 4, "exclusive_gateway_count": 7, "converging_gateway_count": 3, "quality_passed_gates": 25, "quality_failed_gates": 0, "quality_total_gates": 25, "grammar_error_count": 0, "grammar_warning_count": 0, "lint_error_count": 0, "lint_warning_count": 3, "simulation_preview_run_count": 1000, "simulation_preview_step_count": 31, "failed_check_count": 0 } } }, { "run": 3, "started_at": "2026-08-02T22:21:45.378380+00:00", "finished_at": "2026-08-02T22:35:58.336534+00:00", "completed_at": "2026-08-02T22:35:58.336534+00:00", "elapsed_seconds": 852.955, "completion_marker_seen": true, "completion_signal_valid": true, "status": "completed", "completed": true, "failure_code": null, "acceptance_checks": { "completion_marker_present": true, "completion_signal_valid": true, "completion_prose_consistent": true, "definition_key_reported": true, "complete_source_read": true, "draft_create_attempted": true, "draft_create_confirmation_blocked": true, "precreate_semantic_validation_passed": true, "operator_approved_write": true, "definition_update_not_attempted": true, "pending_write_confirmed": true, "definition_retrieved": true, "lint_completed": true, "quality_audit_completed": true, "grammar_audit_completed": true, "simulation_preview_completed": true, "required_read_tools_completed": true, "definition_not_published": true, "zero_factual_clarifications": true, "fixed_model_used": true, "independent_semantic_validation_passed": true }, "turn_metrics": [ { "tool_call_count": 8, "completed_tool_count": 3, "blocked_write_count": 1, "failed_tool_count": 4, "controlled_execution_completed": false, "controlled_execution_declined": false, "precreate_semantic_validation_present": true, "precreate_semantic_validation_passed": true, "precreate_semantic_failed_check_count": 0 }, { "tool_call_count": 6, "completed_tool_count": 6, "blocked_write_count": 0, "failed_tool_count": 0, "controlled_execution_completed": true, "controlled_execution_declined": false, "precreate_semantic_validation_present": false, "precreate_semantic_validation_passed": false, "precreate_semantic_failed_check_count": -1 } ], "clarification_turns": 0, "write_approvals": 1, "tool_call_count": 14, "semantic_validation": { "passed": true, "checks": { "evidence_order_unambiguous": true, "source_attachment_evidence_present": true, "source_attachment_identity_matches": true, "benchmark_source_matches_manifest": true, "source_attachment_extraction_complete": true, "source_attachment_complete_read": true, "source_read_before_definition_write": true, "only_attachment_source_used": true, "all_definition_reads_bound_to_created_definition": true, "controlled_write_chain_evidence_present": true, "create_confirmation_chain_valid": true, "exactly_one_approved_definition_create": true, "no_non_create_definition_mutation_attempted": true, "all_writes_explicitly_confirmed": true, "definition_write_scope_valid": true, "manual_bpmn_xml_not_supplied": true, "validation_evidence_after_last_write": true, "validation_chain_bound_to_final_definition": true, "simulation_preview_result_matches_definition": true, "final_get_after_validations": true, "no_later_definition_mutation": true, "definition_never_published": true, "preparation_subprocess_semantics_exact": true, "semantic_role_mapping_complete": true, "semantic_graph_exact": true, "global_bpmn_element_ids_unique": true, "top_level_routing_probabilities_exact": true, "approval_routing_probabilities_exact": true, "timing_parameters_exact": true, "sla_parameters_exact": true, "required_data_io_exact": true, "data_association_inventory_exact": true, "review_approval_control_semantics_exact": true, "boundary_event_inventory_exact": true, "boundary_event_configuration_exact": true, "multi_instance_inventory_exact": true, "multi_instance_configuration_exact": true, "timing_inventory_exact": true, "sla_inventory_exact": true, "gateway_branch_polarity_exact": true, "event_reference_inventory_exact": true, "non_gateway_flow_semantics_exact": true, "implementation_inventory_exact": true, "activity_descriptions_exact": true, "supplier_crossings_exact": true, "governance_metadata_complete": true, "semantic_contradictions_absent": true, "definition_evidence_present": true, "definition_key_matches": true, "definition_name_matches": true, "definition_description_absent": true, "definition_category_absent": true, "definition_remains_draft": true, "definition_non_executable": true, "definition_uses_simulation_profile": true, "source_pool_identity_exact": true, "participant_minimum_met": true, "external_supplier_present": true, "participant_inventory_exact": true, "participant_accountability_exact": true, "assignee_accountability_exact": true, "message_flow_minimum_met": true, "supplier_message_flows_complete": true, "message_flow_semantics_exact": true, "data_object_minimum_met": true, "required_data_records_present": true, "data_object_inventory_exact": true, "data_object_descriptions_absent": true, "accountable_lane_minimum_met": true, "required_accountable_lanes_present": true, "substantial_graph_present": true, "required_bpmn_elements_present": true, "message_start_and_end_outcomes_present": true, "terminal_outcomes_exact": true, "interrupting_pt48h_escalation_present": true, "sequential_multi_instance_approval_present": true, "scope_exclusions_documented": true, "forbidden_payment_work_absent": true, "excluded_implementation_absent": true, "excluded_scope_contradictions_absent": true, "gateway_probabilities_valid": true, "gateway_defaults_valid": true, "quality_audit_evidence_present": true, "quality_audit_all_gates_passed": true, "quality_audit_includes_gate_14_suite": true, "grammar_audit_evidence_present": true, "grammar_audit_passed": true, "grammar_warnings_clear": true, "lint_evidence_present": true, "lint_passed": true, "lint_simulation_ready": true, "simulation_preview_evidence_present": true, "simulation_preview_non_persisted": true, "simulation_preview_draft_bound": true, "simulation_preview_deterministic_config": true, "simulation_preview_limitations_disclosed": true, "simulation_preview_metrics_finite": true, "simulation_preview_step_coverage_complete": true }, "metrics": { "source_read_page_count": 2, "source_line_count": 139, "write_proposal_count": 1, "write_resolution_count": 1, "approved_write_count": 1, "successful_definition_write_count": 1, "successful_definition_update_count": 0, "bound_validation_count": 4, "participant_count": 2, "message_flow_count": 4, "supplier_message_flow_count": 4, "data_object_count": 6, "accountable_lane_count": 4, "top_level_step_count": 18, "total_step_count": 32, "terminal_outcome_count": 4, "exclusive_gateway_count": 7, "converging_gateway_count": 3, "quality_passed_gates": 25, "quality_failed_gates": 0, "quality_total_gates": 25, "grammar_error_count": 0, "grammar_warning_count": 0, "lint_error_count": 0, "lint_warning_count": 3, "simulation_preview_run_count": 1000, "simulation_preview_step_count": 31, "failed_check_count": 0 } } } ], "publication_note": "Fixed model and fixed-name checks, integer metrics, timings, and aggregate turn counts only. Assistant transcript, failure details, pending execution IDs, definition IDs or keys, source hashes, thread IDs, trace IDs, tool parameters, schemas, proposal summaries, findings, reasoning, credentials, and tenant data are intentionally excluded. Pre-create semantic evidence is reduced to fixed booleans and integer counts. The preview verifies deterministic plumbing only; embedded-subprocess outcome correlation is not yet modeled and the result is not decision-grade operational simulation." }