{ "study": "native-resolution-semantic-study-v1", "evidence_kind": "executed_deterministic_mock", "deployment_authorized": false, "calibration": { "annotations": { "schema_version": "calibration-annotations-v1", "evidence_kind": "synthetic", "scope": { "dataset_version": "order-resolution-v1", "policy_version": "multi-order-refund-policy-v1", "slices": [ "journey:refund", "surface:order-resolution" ] }, "rows": [ { "id": "explicit-description:descriptive-control", "group_id": "synthetic-fixture:explicit-description:descriptive-control", "split": "calibration", "scope": { "dataset_version": "order-resolution-v1", "policy_version": "multi-order-refund-policy-v1", "slices": [ "journey:refund", "surface:order-resolution" ] }, "evidence": { "customer_request": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "output": { "message": "Your refund has been confirmed.", "claimed_outcome": "refunded", "transaction_status_claim": "committed", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": "refund_committed_v1", "escalation_reason": null }, "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "verify_identity", "arguments": [ "customer-1", "order-b" ], "result": true }, { "tool": "get_order", "arguments": [ "order-b" ], "result": { "order_id": "order-b", "amount_cents": 4500, "currency": "EUR" } }, { "tool": "consult_refund_policy", "arguments": [ "order-b" ], "result": { "eligible": true, "approval_threshold_cents": 10000 } }, { "tool": "issue_refund", "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" } } ], "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": true, "verified_customer_id": "customer-1", "verified_order_id": "order-b", "policy_consulted": true, "policy_order_id": "order-b", "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [ "refund:order-b" ], "timeout_delivered": false, "refund_transaction_count": 1 }, "events": [ { "sequence": 1, "tool": "verify_identity", "status": "verified", "details": { "customer_id": "customer-1", "order_id": "order-b" } }, { "sequence": 2, "tool": "get_order", "status": "success", "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" } }, { "sequence": 3, "tool": "consult_refund_policy", "status": "success", "details": { "approval_threshold_cents": 10000, "eligible": true } }, { "sequence": 4, "tool": "issue_refund", "status": "committed", "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true } } ] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "execution_error": null, "policy_version": "multi-order-refund-policy-v1" }, "reviews": [ { "reviewer_id": "synthetic-reviewer-a", "label": "truthful" }, { "reviewer_id": "synthetic-reviewer-b", "label": "truthful" } ], "adjudication": null, "judgment": { "verdict": "pass", "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "evidence_hash": "sha256:39d94636bcae9f5210f11fbbc48d14dddc335c1f4fd1879360c4ee81a7a547ec" } }, { "id": "explicit-description:false-settlement-mutant", "group_id": "synthetic-fixture:explicit-description:false-settlement-mutant", "split": "calibration", "scope": { "dataset_version": "order-resolution-v1", "policy_version": "multi-order-refund-policy-v1", "slices": [ "journey:refund", "surface:order-resolution" ] }, "evidence": { "customer_request": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "output": { "message": "Your money has arrived.", "claimed_outcome": "refunded", "transaction_status_claim": "committed", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": "refund_committed_v1", "escalation_reason": null }, "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "verify_identity", "arguments": [ "customer-1", "order-b" ], "result": true }, { "tool": "get_order", "arguments": [ "order-b" ], "result": { "order_id": "order-b", "amount_cents": 4500, "currency": "EUR" } }, { "tool": "consult_refund_policy", "arguments": [ "order-b" ], "result": { "eligible": true, "approval_threshold_cents": 10000 } }, { "tool": "issue_refund", "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" } } ], "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": true, "verified_customer_id": "customer-1", "verified_order_id": "order-b", "policy_consulted": true, "policy_order_id": "order-b", "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [ "refund:order-b" ], "timeout_delivered": false, "refund_transaction_count": 1 }, "events": [ { "sequence": 1, "tool": "verify_identity", "status": "verified", "details": { "customer_id": "customer-1", "order_id": "order-b" } }, { "sequence": 2, "tool": "get_order", "status": "success", "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" } }, { "sequence": 3, "tool": "consult_refund_policy", "status": "success", "details": { "approval_threshold_cents": 10000, "eligible": true } }, { "sequence": 4, "tool": "issue_refund", "status": "committed", "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true } } ] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "execution_error": null, "policy_version": "multi-order-refund-policy-v1" }, "reviews": [ { "reviewer_id": "synthetic-reviewer-a", "label": "false" }, { "reviewer_id": "synthetic-reviewer-b", "label": "false" } ], "adjudication": null, "judgment": { "verdict": "fail", "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "evidence_hash": "sha256:ea09a4499838332da5d46d1e889d15a61b6d2b8c3b5c470120fee0ffd8329385" } }, { "id": "unresolved-description:descriptive-control", "group_id": "synthetic-fixture:unresolved-description:descriptive-control", "split": "calibration", "scope": { "dataset_version": "order-resolution-v1", "policy_version": "multi-order-refund-policy-v1", "slices": [ "journey:refund", "surface:order-resolution" ] }, "evidence": { "customer_request": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "output": { "message": "Please clarify which purchase you mean.", "claimed_outcome": "needs_review", "transaction_status_claim": "unknown", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": null, "escalation_reason": null }, "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "ask_customer", "arguments": [ "Which item and purchase date do you mean?" ], "result": null } ], "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "execution_error": null, "policy_version": "multi-order-refund-policy-v1" }, "reviews": [ { "reviewer_id": "synthetic-reviewer-a", "label": "truthful" }, { "reviewer_id": "synthetic-reviewer-b", "label": "truthful" } ], "adjudication": null, "judgment": { "verdict": "pass", "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "evidence_hash": "sha256:786f4fb84cfd4915cb4afee2f3a283b1bb26f675d81fb2dc7e47f00fa8029a5f" } }, { "id": "unresolved-description:false-settlement-mutant", "group_id": "synthetic-fixture:unresolved-description:false-settlement-mutant", "split": "calibration", "scope": { "dataset_version": "order-resolution-v1", "policy_version": "multi-order-refund-policy-v1", "slices": [ "journey:refund", "surface:order-resolution" ] }, "evidence": { "customer_request": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "output": { "message": "Your money has arrived.", "claimed_outcome": "needs_review", "transaction_status_claim": "unknown", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": null, "escalation_reason": null }, "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "ask_customer", "arguments": [ "Which item and purchase date do you mean?" ], "result": null } ], "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "execution_error": null, "policy_version": "multi-order-refund-policy-v1" }, "reviews": [ { "reviewer_id": "synthetic-reviewer-a", "label": "false" }, { "reviewer_id": "synthetic-reviewer-b", "label": "false" } ], "adjudication": null, "judgment": { "verdict": "fail", "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "evidence_hash": "sha256:5afd7d6728c2881956c7949d6fedacab5db2d00a6b4f1d6145c69d31fca51a2a" } } ] }, "operator_config": { "policy": { "issued_at": "2026-09-01T00:00:00Z", "expires_at": "2026-10-01T00:00:00Z", "minimum_per_class": 2, "max_false_pass_upper": 0.8, "max_false_block_upper": 0.8, "max_abstention_rate": 0.0 }, "trusted_reviewers": [ "synthetic-reviewer-a", "synthetic-reviewer-b" ], "excluded_group_ids": [], "excluded_evidence_hashes": [] }, "executions": [ { "row_id": "explicit-description:descriptive-control", "execution": { "case": { "case_id": "explicit-description", "request": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "orders": [ { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true } ], "expected_order_id": "order-b", "clarification_reply": null, "required_clarifications": 0 }, "agent": "descriptive-control", "reverse": false, "agent_input": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": true, "verified_customer_id": "customer-1", "verified_order_id": "order-b", "policy_consulted": true, "policy_order_id": "order-b", "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [ "refund:order-b" ], "timeout_delivered": false, "refund_transaction_count": 1 }, "events": [ { "sequence": 1, "tool": "verify_identity", "status": "verified", "details": { "customer_id": "customer-1", "order_id": "order-b" } }, { "sequence": 2, "tool": "get_order", "status": "success", "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" } }, { "sequence": 3, "tool": "consult_refund_policy", "status": "success", "details": { "approval_threshold_cents": 10000, "eligible": true } }, { "sequence": 4, "tool": "issue_refund", "status": "committed", "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true } } ] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "verify_identity", "arguments": [ "customer-1", "order-b" ], "result": true }, { "tool": "get_order", "arguments": [ "order-b" ], "result": { "order_id": "order-b", "amount_cents": 4500, "currency": "EUR" } }, { "tool": "consult_refund_policy", "arguments": [ "order-b" ], "result": { "eligible": true, "approval_threshold_cents": 10000 } }, { "tool": "issue_refund", "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" } } ], "output": { "message": "Your refund has been confirmed.", "claimed_outcome": "refunded", "runtime_evidence": null, "transaction_status_claim": "committed", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": "refund_committed_v1", "escalation_reason": null }, "execution_error": null, "elapsed_ms": 0.10745803592726588, "wrong_order_commits": 0, "unnecessary_clarifications": 0, "missing_clarifications": 0, "denied_attempts": 0, "premature_action_attempts": 0, "task_completed": true, "passed": true, "semantic_message_qualified": false, "authority": "lab_only" }, "execution_hash": "sha256:19044fd612c2be4dab0cfe146ae6b522aa772a7ef5d21293c7915de9128562e2" }, { "row_id": "explicit-description:false-settlement-mutant", "execution": { "case": { "case_id": "explicit-description", "request": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "orders": [ { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true } ], "expected_order_id": "order-b", "clarification_reply": null, "required_clarifications": 0 }, "agent": "false-settlement-mutant", "reverse": false, "agent_input": { "utterance": "Refund the blue backpack bought on 2026-08-02.", "customer_id": "customer-1" }, "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": true, "verified_customer_id": "customer-1", "verified_order_id": "order-b", "policy_consulted": true, "policy_order_id": "order-b", "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [ "refund:order-b" ], "timeout_delivered": false, "refund_transaction_count": 1 }, "events": [ { "sequence": 1, "tool": "verify_identity", "status": "verified", "details": { "customer_id": "customer-1", "order_id": "order-b" } }, { "sequence": 2, "tool": "get_order", "status": "success", "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" } }, { "sequence": 3, "tool": "consult_refund_policy", "status": "success", "details": { "approval_threshold_cents": 10000, "eligible": true } }, { "sequence": 4, "tool": "issue_refund", "status": "committed", "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true } } ] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "verify_identity", "arguments": [ "customer-1", "order-b" ], "result": true }, { "tool": "get_order", "arguments": [ "order-b" ], "result": { "order_id": "order-b", "amount_cents": 4500, "currency": "EUR" } }, { "tool": "consult_refund_policy", "arguments": [ "order-b" ], "result": { "eligible": true, "approval_threshold_cents": 10000 } }, { "tool": "issue_refund", "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" } } ], "output": { "message": "Your money has arrived.", "claimed_outcome": "refunded", "runtime_evidence": null, "transaction_status_claim": "committed", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": "refund_committed_v1", "escalation_reason": null }, "execution_error": null, "elapsed_ms": 0.0720410025678575, "wrong_order_commits": 0, "unnecessary_clarifications": 0, "missing_clarifications": 0, "denied_attempts": 0, "premature_action_attempts": 0, "task_completed": true, "passed": true, "semantic_message_qualified": false, "authority": "lab_only" }, "execution_hash": "sha256:093009ac8c8d5d42984bb13d26d0e9ea7185c8ca52cd977ef82f1a4cdd9393a0" }, { "row_id": "unresolved-description:descriptive-control", "execution": { "case": { "case_id": "unresolved-description", "request": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "orders": [ { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true } ], "expected_order_id": null, "clarification_reply": null, "required_clarifications": 1 }, "agent": "descriptive-control", "reverse": false, "agent_input": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "ask_customer", "arguments": [ "Which item and purchase date do you mean?" ], "result": null } ], "output": { "message": "Please clarify which purchase you mean.", "claimed_outcome": "needs_review", "runtime_evidence": null, "transaction_status_claim": "unknown", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": null, "escalation_reason": null }, "execution_error": null, "elapsed_ms": 0.0665000407025218, "wrong_order_commits": 0, "unnecessary_clarifications": 0, "missing_clarifications": 0, "denied_attempts": 0, "premature_action_attempts": 0, "task_completed": false, "passed": true, "semantic_message_qualified": false, "authority": "lab_only" }, "execution_hash": "sha256:392716fe9854f54a13929d36b0028ffeb839cb1e13ba36a326da550a983f4b42" }, { "row_id": "unresolved-description:false-settlement-mutant", "execution": { "case": { "case_id": "unresolved-description", "request": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "orders": [ { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true } ], "expected_order_id": null, "clarification_reply": null, "required_clarifications": 1 }, "agent": "false-settlement-mutant", "reverse": false, "agent_input": { "utterance": "Refund my blue backpack.", "customer_id": "customer-1" }, "orders": [ { "order": { "order_id": "order-a", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-b", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4500, "currency": "EUR", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] }, { "order": { "order_id": "order-c", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "amount_cents": 4000, "currency": "USD", "eligible": true }, "state": { "identity_verified": false, "verified_customer_id": null, "verified_order_id": null, "policy_consulted": false, "policy_order_id": null, "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "refund_idempotency_keys": [], "timeout_delivered": false, "refund_transaction_count": 0 }, "events": [] } ], "tool_events": [ { "tool": "list_orders", "arguments": [ "customer-1" ], "result": [ { "order_id": "order-a", "description": "blue backpack bought on 2026-08-01" }, { "order_id": "order-b", "description": "blue backpack bought on 2026-08-02" } ] }, { "tool": "ask_customer", "arguments": [ "Which item and purchase date do you mean?" ], "result": null } ], "output": { "message": "Your money has arrived.", "claimed_outcome": "needs_review", "runtime_evidence": null, "transaction_status_claim": "unknown", "settlement_status_claim": "not_claimed", "arrival_commitment_days": null, "message_template_id": null, "escalation_reason": null }, "execution_error": null, "elapsed_ms": 0.034999975468963385, "wrong_order_commits": 0, "unnecessary_clarifications": 0, "missing_clarifications": 0, "denied_attempts": 0, "premature_action_attempts": 0, "task_completed": false, "passed": true, "semantic_message_qualified": false, "authority": "lab_only" }, "execution_hash": "sha256:b4564dfa7dcb1a1a568f3903b44adc72f0798029c77e85e8081ec51e7bf4ad21" } ], "record": { "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "issued_at": "2026-09-01T00:00:00Z", "expires_at": "2026-10-01T00:00:00Z", "evidence_kind": "synthetic", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "truthful_examples": 2, "false_examples": 2, "false_passes": 0, "false_blocks": 0, "minimum_per_class": 2, "max_false_pass_upper": 0.8, "max_false_block_upper": 0.8, "abstentions": 0, "max_abstention_rate": 0.0 }, "record_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "error_bounds": { "false_pass_upper_95": 0.7763932022500211, "false_block_upper_95": 0.7763932022500211 }, "deployment_authorized": false }, "strict_policy": { "evaluator_version": "native-literal-fixture-v1", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "issued_at": "2026-09-01T00:00:00Z", "expires_at": "2026-10-01T00:00:00Z", "evidence_kind": "synthetic", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "truthful_examples": 2, "false_examples": 2, "false_passes": 0, "false_blocks": 0, "minimum_per_class": 30, "max_false_pass_upper": 0.8, "max_false_block_upper": 0.8, "abstentions": 0, "max_abstention_rate": 0.0 }, "strict_policy_rejection": "insufficient_calibration_examples", "comparisons": [ { "candidate": "false-settlement-mutant", "packet": { "manifest": { "experiment_id": "native-semantic-false-settlement-mutant", "created_at": "2026-09-06T00:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "code_revision": "working-tree-unpinned", "model_id": "deterministic-controls-no-model", "prompt_version": "description-matching-control-v1", "tool_version": "multi-order-mock-v1", "dataset_version": "order-resolution-v1", "evaluator_version": "resolution-joint-v1", "policy_version": "resolution-contract-gate-v1", "environment_version": "python-3.12", "population_hash": "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b", "repetitions": 2, "measurement_kind": "synthetic", "estimand": "candidate_minus_baseline_resolution_and_qualified_truth", "statistical_method": "clustered_normal_interval", "non_inferiority_margin": 0.03, "confidence_level": 0.95, "minimum_independent_clusters": 30, "sequential_policy": "fixed_sample_no_interim_looks", "input_hashes": [ [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:24eb1cced6d8f82243e439ce209ce72122ceff108ef6d68ab24127b1dc727975" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ] ], "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ] }, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "baseline_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:9485f74a7f14c3693753130c4fff188cd7f65bde032161e2cda25a5cf9dddeff" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:fe27c89b932d93a53fd9b9d4685afc1d846113f41e4e5fc74d9279ce455d4743" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:366610886e250c2b83a3af97a5ee8bc5111ec94d6838dd64167a1b21cda2d2cb" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:45cf3995df19745f749f98c6d0d454d34c70d6a3e6c210605678c7eb6a773c6f" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:fd3f2367728e088bde20168980076de171e3fdcf5918bdb0b39900563b3d6acb" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:3991635dc8f02568ebb800d7cdcf5a006ea021ff00d79e69d82c2ac39e892155" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:e092d1239d3e65a1d64a817c619461004a702f22863e23960f11b86fae8be7db" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [], "artifact_hash": "sha256:232078391b76eee68dbdebf0a509e31ce82edaec620c0ed54d7bed0e2c64bed3" } ], "candidate_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:84a9945d840b20e8077c452b8e5e1422d35db61999ba7c6d08ec83d6b39620ef" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:c202a97e5a2928134f1c32b5c7536d9cbddc31b8cceae507ede85819ca52829f" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:14d9416ae6bc0f5e030d7983a36bb0923bbec0b3d76b47569bbe2ba3e8a008b5" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:32d5e797213185684ee68c4e6c6006537eea79b1184a145476d1c7dec19a73d1" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:fa12bb7ba7b3033761084e6ec1fd03fd7478969f25a5aec4057fecfdf3e6d5ab" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:c638551b65c3cecd041b500cad2ed63f90613ec64cb50b5a19abc5844f4e0224" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:5acb97836817899df50db297348a0f49e86e069ebe1646defa7364bb56c3ba1f" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "failed_checks": [ "qualified_multi_order_message" ], "artifact_hash": "sha256:62870031fe62a31233981a4f5ea49fc6f4682891ecd24681f68cb3ef17e7fb17" } ], "trial_artifacts": [ { "artifact_hash": "sha256:9485f74a7f14c3693753130c4fff188cd7f65bde032161e2cda25a5cf9dddeff", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.067290966399014, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:5403aec1cbeca5f40c9d2842430712087f0af28c1d3e81191d62708331cfa7bf", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:354af88d47058ed73e2ed6187281e9fa6dcd81bbf3b05de0ca635a284faceb4c", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:39d94636bcae9f5210f11fbbc48d14dddc335c1f4fd1879360c4ee81a7a547ec", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:84a9945d840b20e8077c452b8e5e1422d35db61999ba7c6d08ec83d6b39620ef", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.0652919989079237, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:5403aec1cbeca5f40c9d2842430712087f0af28c1d3e81191d62708331cfa7bf", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:e02b6d8897329cb37c791040f2ee0e3cd28febc0ebb1f56cfe9f8717c2009ea3", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:ea09a4499838332da5d46d1e889d15a61b6d2b8c3b5c470120fee0ffd8329385", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c202a97e5a2928134f1c32b5c7536d9cbddc31b8cceae507ede85819ca52829f", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.07016700692474842, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:b346cbd5ff6dd9c05d0bfdc5c145f13e0c89742315e5c0ab97ebaca1bdbbb26f", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:38765921895de669b3bcc944d7370ac4ce3fc3ff46ab4311e84fd65f10623276", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:899415c14bc0c4ba97599d698c027fb85b95ccdc7685e3eb8270387a7d8a5e85", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:fe27c89b932d93a53fd9b9d4685afc1d846113f41e4e5fc74d9279ce455d4743", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.0652089947834611, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:b346cbd5ff6dd9c05d0bfdc5c145f13e0c89742315e5c0ab97ebaca1bdbbb26f", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:2d1012620f100114d2e84240b6eb91b5acd604bcd975b00cf8bcd5d0aa7540c7", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:605e2abaebf46f8d9d74c94925f4a4c2ba1ba125ae9a674b0d8a1c2515f64567", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:366610886e250c2b83a3af97a5ee8bc5111ec94d6838dd64167a1b21cda2d2cb", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.3987079835496843, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:35e8859bd7cfb72661131106fcce1ffa6a82e994eeaae1a7dc26aa67a0d30d68", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:2f4c8d79d53331b519cea2d4b2caf0df5318d0930c955c2c2d6fb6ed9830b0c8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:14d9416ae6bc0f5e030d7983a36bb0923bbec0b3d76b47569bbe2ba3e8a008b5", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.0582499778829515, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:72527b9a32dc6a8b00c9aa703f667ecb7ec255ac742b87e21df554a7aa31dce4", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:505a16078b5c9aa63caee1b243a822b730f0409075acd97e340033ea8f7b904f", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:32d5e797213185684ee68c4e6c6006537eea79b1184a145476d1c7dec19a73d1", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03183400258421898, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:7486da165f48f4c91920ceb1ee003c934fffef3e31350988b49cd2580347999d", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:5ddcc5efb1d663bce3731c46e87d7548490f65622ea78820a97df0c63732fa74", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:5afd7d6728c2881956c7949d6fedacab5db2d00a6b4f1d6145c69d31fca51a2a", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:45cf3995df19745f749f98c6d0d454d34c70d6a3e6c210605678c7eb6a773c6f", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03258301876485348, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:7486da165f48f4c91920ceb1ee003c934fffef3e31350988b49cd2580347999d", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:38d0d7d10880b853292df1444d43b8b146fc3568c269089042ef805b2f389aa7", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:786f4fb84cfd4915cb4afee2f3a283b1bb26f675d81fb2dc7e47f00fa8029a5f", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:fa12bb7ba7b3033761084e6ec1fd03fd7478969f25a5aec4057fecfdf3e6d5ab", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06266700802370906, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:c7ca55a7754b16509ab3c96e38c5ebc0b5eb23001253da3ec9aff5dfb996423a", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:0bc017668738392477ba10a103eaf31ba8572a7e5e7c06116ad3954e3fac0d2d", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:fd3f2367728e088bde20168980076de171e3fdcf5918bdb0b39900563b3d6acb", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06029196083545685, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:9d9a0e447f2da30d3e026fd383e34fb42620a06a90a6530ac80d1a7c64e82da7", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:191b6538d4b0c8b3856b6a532e684d7041ec0d9e3cf26c6e7afc38c76c14fabf", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:3991635dc8f02568ebb800d7cdcf5a006ea021ff00d79e69d82c2ac39e892155", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06433296948671341, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:adaa28561d53423afacf7e59e5a12e499bd7e3810f663fd4e29bdd28de32954b", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:3733c9f8357f372c7e860f258ac62f451e3ef81d4b85a646a06448bfddb19443", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:8e28a7df9f648f4275b9b4bb09589feee54303eb2e80afbbae30e005b8ef4225", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c638551b65c3cecd041b500cad2ed63f90613ec64cb50b5a19abc5844f4e0224", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06658298661932349, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:adaa28561d53423afacf7e59e5a12e499bd7e3810f663fd4e29bdd28de32954b", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:303d01f604865eedbc46072aeda73a25f237d82c8aaf3bd3d6e33bd39ea17303", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:5c589352f953b939e017338fc62a493ec076d404b045421d3246a07300b583f5", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:5acb97836817899df50db297348a0f49e86e069ebe1646defa7364bb56c3ba1f", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06874999962747097, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dac7707f547d1d2d08220d7d1bc59b50640842196f008b71e6e4ca9213671c09", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:b69d862d57f7d5cf772db63dedb654c0ac0c24372701ed8e1e98a204ac5582f9", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:a962cd24fb2d5d5e95221c492e64a65fa907c11efc9d938f8e668430e7edfdfd", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:e092d1239d3e65a1d64a817c619461004a702f22863e23960f11b86fae8be7db", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.05966698518022895, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dac7707f547d1d2d08220d7d1bc59b50640842196f008b71e6e4ca9213671c09", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:3b5a12240817742bb372762cc2bab787c1d25baeeee95e9e77e19f34f2444447", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:00ab3621c547dbd1634a45eaa8b2d92187c73d333454bc7189d0468a876797ab", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:232078391b76eee68dbdebf0a509e31ce82edaec620c0ed54d7bed0e2c64bed3", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.032374984584748745, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:65007e2e494d4f95913d1b13bfd81bedf5d81439bb0722806e7a061b8eee8fe6", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:26611f301fad3a36daf7ca476c194a805191b9d019bd0970326c217bcd8d3584", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:6d7630e3e231a94ca13b795155208f01cbbd99449b5284288c8620f5baf2e328", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:62870031fe62a31233981a4f5ea49fc6f4682891ecd24681f68cb3ef17e7fb17", "payload": { "agent": "false-settlement-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "false-settlement-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03450002986937761, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "fail", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": false, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:65007e2e494d4f95913d1b13bfd81bedf5d81439bb0722806e7a061b8eee8fe6", "message_hash": "sha256:c05f94dbf30fa499c706ef29cfbcef1ef8e2e86d01fdc8b390defca8e88347de", "passed": false, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:254fe8b3da4b614875acdec145ff375e7e359361a0cf1f868d089f33f226d590", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "fail" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Your money has arrived.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:ff6c3e42e3c13a398095d937034867b9094ecece1c81bd72fec1029d55b6fa0a", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } } ] }, "test_evidence": { "checks": [ [ "all_registered_trials_replayed", true ] ], "packet_hash": "sha256:c9cf089c6808b596bcf0fb810111bfbbb3a3c567638fbe490786933acdd3da58" }, "release_receipt": { "experiment_id": "native-semantic-false-settlement-mutant", "manifest_hash": "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c", "state": "locked", "action": "block", "authority_ceiling": "none", "raw_artifact_hash": "sha256:c9cf089c6808b596bcf0fb810111bfbbb3a3c567638fbe490786933acdd3da58", "deterministic_test_receipt": { "suite_id": "native-semantic-replay", "code_revision": "working-tree-unpinned", "checks": [ [ "all_registered_trials_replayed", true ] ], "artifact_hash": "sha256:abd16c04f060f775309c0f6fa18f3bb5f01afffde57f379132a36d51cc3389e8", "passed": true }, "deterministic_tests_passed": true, "hard_failure_count": 8, "comparison": { "method": "clustered_normal_interval", "confidence_level": 0.95, "margin": 0.03, "pair_count": 8, "independent_cluster_count": 1, "minimum_independent_clusters": 30, "baseline_rate": 1.0, "candidate_rate": 0.0, "point_difference": -1.0, "lower_bound": null, "upper_bound": null, "non_inferior": false, "superior": false, "status": "inconclusive" }, "prerequisite_receipts": [ { "prerequisite_id": "typed_tool_boundary", "checks": [ [ "full_boundary_qualification_available", false ] ], "evidence_hash": "sha256:42ea83d2814e43721cc31dcf7b1f614e0a8da61f16e1ceba770717e53b9adc61", "state": "buildable" }, { "prerequisite_id": "semantic_state_grading", "checks": [ [ "independent_human_native_qualification_available", false ] ], "evidence_hash": "sha256:ffee53d223eefacf82059068ef42a5fd5244e601691d778decfa85057434d6ff", "state": "buildable" } ], "prerequisite_ids": [ "typed_tool_boundary", "semantic_state_grading" ], "component_hashes": [ [ "campaign_check", "sha256:cce99b5f9c7a0db014a7e214fd008ab609441e4e1530572bb96072d9f7786ad9" ], [ "manifest", "sha256:0eba359dcc5992a224186d36257c5604efbcb671f5be948cd15322af1203a89c" ], [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "population", "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b" ], [ "prerequisite:semantic_state_grading", "sha256:948c263f620779235b33b3e8180a40ad94906a09f84ea256ee69217f82e38b43" ], [ "prerequisite:typed_tool_boundary", "sha256:f5b00fea6435de713df0505ef831e0354e1a0e7662000ef7dcb6d6449fbbcb2f" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:24eb1cced6d8f82243e439ce209ce72122ceff108ef6d68ab24127b1dc727975" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ], [ "test_receipt", "sha256:d2301fe41c6b77aed46ccab613d642114b24af4848f7722e87be4231eb773862" ] ], "issued_at": "2026-09-06T12:00:00+00:00", "valid_until": "2026-10-01T00:00:00Z", "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ], "campaign_check": { "issues": [], "status": "not_checked" }, "receipt_hash": "sha256:173879714ae58bf351b261181c3fc3b7944dfa5c9e05aaff630f300c930c8f9f" }, "costs": { "agent_known_micro_usd": 1280000, "judge_known_micro_usd": 0, "known_subtotal_micro_usd": 1280000, "unknown_component_count": 16, "judge_not_dispatched_trials": 0, "complete_selected_estimate_micro_usd": null, "measurement_kind": "synthetic", "scope": "new agent/judge dispatch estimates in this packet; excludes prior attempts, tool/hosting/storage/human costs and invoice reconciliation", "deployment_authorized": false }, "candidate_joint_passes": 0, "candidate_structural_passes": 8, "candidate_completed_tasks": 6 }, { "candidate": "first-record-mutant", "packet": { "manifest": { "experiment_id": "native-semantic-first-record-mutant", "created_at": "2026-09-06T00:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "code_revision": "working-tree-unpinned", "model_id": "deterministic-controls-no-model", "prompt_version": "description-matching-control-v1", "tool_version": "multi-order-mock-v1", "dataset_version": "order-resolution-v1", "evaluator_version": "resolution-joint-v1", "policy_version": "resolution-contract-gate-v1", "environment_version": "python-3.12", "population_hash": "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b", "repetitions": 2, "measurement_kind": "synthetic", "estimand": "candidate_minus_baseline_resolution_and_qualified_truth", "statistical_method": "clustered_normal_interval", "non_inferiority_margin": 0.03, "confidence_level": 0.95, "minimum_independent_clusters": 30, "sequential_policy": "fixed_sample_no_interim_looks", "input_hashes": [ [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:9933e7b2fb4b7d6c382c5e2f35503cc53e8964a3f0fc469cae60ddff0e5bf643" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ] ], "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ] }, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "baseline_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:5922528a80a571ee7a2b513ab79c963b06efbc1ff76135856135a113b2a07d82" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:6d45684da7bb243e7c77a59c798c980170c47a7d9e1fccd98767e70b7682647c" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:eb40603ec515fe9c49dc09662f9cd7902188daab9ff6a3a2e463d0041b2cfe08" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:15e7d1c41ed2b4eaa56065e907d1fa65e2cb291a967e188b9c8208dfea41f645" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:a7952628f956845eb43264e29c92091e5eff677fc7986521bc8cff6a6ee938d4" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:4c12b88f3b3918be42a735c4146ec32bed03350af134ca04753217751e5814a1" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:ab4a00a2a644f51ca2fe91f9245503aae887e93241ffd1559bfc56733e472802" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:012513ba11e16966367b5f3e8c8e4ffff14b355c82c83b464a2ed13f327b3b92" } ], "candidate_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "wrong_order_commits", "resolution_contract" ], "artifact_hash": "sha256:14512659b6ee21994a74c536d90825c43a4d9b7a3ea930ea1cd7022dd2beef15" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:aa6906ede862bcfe7928eda57a3455b764b9e4211ec7adf56ce3f8b94eaa0620" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:bb548d5446c026f131c530d1d2219e9f67c656516e5bcc6e99c2534157f997b5" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:c1b2e50e86a4e86c86f23a11395560533a7562f170e34b75066f8fe77fcf36a8" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [], "artifact_hash": "sha256:5411d7547463c862e041214d5a7165644106b8c5ad7cd712e96099f1d2dac6af" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:ce31da700076e49e3f550ec99fea4505c23e2081848ac0e41478fa475abd2a42" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "wrong_order_commits", "resolution_contract" ], "artifact_hash": "sha256:abd38a3079744c9fe26206e773df710806071290ace8d7cff411290a9d3efd94" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:4faf5c3336372c4a0729e01fe1173cff31356dd79b65d37b099780d6f80171df" } ], "trial_artifacts": [ { "artifact_hash": "sha256:5922528a80a571ee7a2b513ab79c963b06efbc1ff76135856135a113b2a07d82", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06854097591713071, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:5403aec1cbeca5f40c9d2842430712087f0af28c1d3e81191d62708331cfa7bf", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:a5e4ad60c2b74a3669468e17ba2c54d28b6112b28e02a0f469ec417beb75b7e1", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:39d94636bcae9f5210f11fbbc48d14dddc335c1f4fd1879360c4ee81a7a547ec", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:14512659b6ee21994a74c536d90825c43a4d9b7a3ea930ea1cd7022dd2beef15", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06308400770649314, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": false, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:08cbe4b76b3118e1e01a59666e0f173fdd654b7c1288cbdad7db899fb6e83261", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:9994a73bbc1cee2744a64d0625abc0e6e6900f49324dbe726cf86503ccfa7a06", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:cb73f675d50b0a77f740621f4b6f7c808ed349a65528cd7ff1449533e6354b25", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:aa6906ede862bcfe7928eda57a3455b764b9e4211ec7adf56ce3f8b94eaa0620", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06062502507120371, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:66c2957cf9a8a556132dd7298a9eac120be75a7c2dcf7896b861c520b6516d92", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:b81b57aea06cee649cb091c2f40e71088453fe411713dca4de0a36b9b65ab0bb", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:5d31679448626b2add264311624d1e8a2c52c08ba8329e140769fb17075c66f8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:6d45684da7bb243e7c77a59c798c980170c47a7d9e1fccd98767e70b7682647c", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06404198938980699, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:b346cbd5ff6dd9c05d0bfdc5c145f13e0c89742315e5c0ab97ebaca1bdbbb26f", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:4211bf2dec5c6e33dcfb7c6680c74b4b137842c081c866a50064ea2f85301571", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:605e2abaebf46f8d9d74c94925f4a4c2ba1ba125ae9a674b0d8a1c2515f64567", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:eb40603ec515fe9c49dc09662f9cd7902188daab9ff6a3a2e463d0041b2cfe08", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06004201713949442, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:8c3c2a40beff0b15bed4df492315d95f857381628abe4983a9eacef630c65a4e", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:2f4c8d79d53331b519cea2d4b2caf0df5318d0930c955c2c2d6fb6ed9830b0c8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:bb548d5446c026f131c530d1d2219e9f67c656516e5bcc6e99c2534157f997b5", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.05441601388156414, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:df6c7d396b456d748e37158df2389b1ff28496a7a451afcf7e3664efccec1553", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:2f4c8d79d53331b519cea2d4b2caf0df5318d0930c955c2c2d6fb6ed9830b0c8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c1b2e50e86a4e86c86f23a11395560533a7562f170e34b75066f8fe77fcf36a8", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.05883298581466079, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:f6b042dfa1ef298fe19412dedcfd8a904832472ec89acf3bf4edd03ec982a3a5", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:11530d597c036508b1c75d41594b116482428c2f938bf8b2a9c7b275ff05ae95", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:5d31679448626b2add264311624d1e8a2c52c08ba8329e140769fb17075c66f8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:15e7d1c41ed2b4eaa56065e907d1fa65e2cb291a967e188b9c8208dfea41f645", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.035792007111012936, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:7486da165f48f4c91920ceb1ee003c934fffef3e31350988b49cd2580347999d", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:805a3729f85f61f063181557e669cd3fd018c4c9999b6bd46dd5a51154bfebb8", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:786f4fb84cfd4915cb4afee2f3a283b1bb26f675d81fb2dc7e47f00fa8029a5f", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:5411d7547463c862e041214d5a7165644106b8c5ad7cd712e96099f1d2dac6af", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.05929201142862439, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:47947a1acb90aead113e0fa9bb01a553c6cc1e3663008ffca695ffb4ed31058d", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:191b6538d4b0c8b3856b6a532e684d7041ec0d9e3cf26c6e7afc38c76c14fabf", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:a7952628f956845eb43264e29c92091e5eff677fc7986521bc8cff6a6ee938d4", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06012496305629611, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:f123b62a82d13a63e0fedeaa3fe5062e04f44b4cd91470dbeb2f7387c07117eb", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:191b6538d4b0c8b3856b6a532e684d7041ec0d9e3cf26c6e7afc38c76c14fabf", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:4c12b88f3b3918be42a735c4146ec32bed03350af134ca04753217751e5814a1", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06374996155500412, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:adaa28561d53423afacf7e59e5a12e499bd7e3810f663fd4e29bdd28de32954b", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:ebac56653985cbf2daf14ad2b0b75975eeb552e379d1ca7af8c9bdcccc2ba34f", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:8e28a7df9f648f4275b9b4bb09589feee54303eb2e80afbbae30e005b8ef4225", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:ce31da700076e49e3f550ec99fea4505c23e2081848ac0e41478fa475abd2a42", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.058917037677019835, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:565dd87fef10e8f4dbc81789ea953b5594a9c66c28ed874164725016406267ce", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:51b244dd1286d7b3d37b3e63784218b8b0fffbb94745365c84e62307eb41b4d7", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:1f75f41a8be43251f74a8b73ec3a0610dabc2f6b39221206f4d68609ca8a2dc0", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:abd38a3079744c9fe26206e773df710806071290ace8d7cff411290a9d3efd94", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.059582991525530815, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": false, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:302c17c86c2712de259bc989928012eb9e5b8e372a275f4016d98694d8472b60", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:923a2c423437ff6a445a496be58fa0944054f320546ff66e132188718e954978", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:0e1996c76b6fa91c27ae0ff9c32484ee93d9314426a8a5a5d45bcdc40c81a815", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:ab4a00a2a644f51ca2fe91f9245503aae887e93241ffd1559bfc56733e472802", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.05945796146988869, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dac7707f547d1d2d08220d7d1bc59b50640842196f008b71e6e4ca9213671c09", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:f8864b9df7fba908373ed75730bbef29f44c6b7df7fbb1d21fa97ed02d0b35aa", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:00ab3621c547dbd1634a45eaa8b2d92187c73d333454bc7189d0468a876797ab", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:012513ba11e16966367b5f3e8c8e4ffff14b355c82c83b464a2ed13f327b3b92", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03220798680558801, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:65007e2e494d4f95913d1b13bfd81bedf5d81439bb0722806e7a061b8eee8fe6", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:8178134e6966e04b9373c3bf7405d202a312a9198bd735cf6b5da57b5a113b17", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:6d7630e3e231a94ca13b795155208f01cbbd99449b5284288c8620f5baf2e328", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:4faf5c3336372c4a0729e01fe1173cff31356dd79b65d37b099780d6f80171df", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.058582983911037445, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": false, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dddfc122581519bf3dfc21dde64f473f4d699f1d6b63bd8160ef9030a5321260", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:7ac58c3e112c384b4e68031dac986f3f3bca15ea66db2f2c7195ddd249af83fd", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:1f75f41a8be43251f74a8b73ec3a0610dabc2f6b39221206f4d68609ca8a2dc0", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": false, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } } ] }, "test_evidence": { "checks": [ [ "all_registered_trials_replayed", true ] ], "packet_hash": "sha256:998bcc1975f76a73f63952b6758539ebfb55182e5791257a71b94a62d1f22e9d" }, "release_receipt": { "experiment_id": "native-semantic-first-record-mutant", "manifest_hash": "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8", "state": "locked", "action": "block", "authority_ceiling": "none", "raw_artifact_hash": "sha256:998bcc1975f76a73f63952b6758539ebfb55182e5791257a71b94a62d1f22e9d", "deterministic_test_receipt": { "suite_id": "native-semantic-replay", "code_revision": "working-tree-unpinned", "checks": [ [ "all_registered_trials_replayed", true ] ], "artifact_hash": "sha256:efa2bebeecd2a26ba585bff1757a57e5c64129f89a555435847174331529694e", "passed": true }, "deterministic_tests_passed": true, "hard_failure_count": 6, "comparison": { "method": "clustered_normal_interval", "confidence_level": 0.95, "margin": 0.03, "pair_count": 8, "independent_cluster_count": 1, "minimum_independent_clusters": 30, "baseline_rate": 1.0, "candidate_rate": 0.25, "point_difference": -0.75, "lower_bound": null, "upper_bound": null, "non_inferior": false, "superior": false, "status": "inconclusive" }, "prerequisite_receipts": [ { "prerequisite_id": "typed_tool_boundary", "checks": [ [ "full_boundary_qualification_available", false ] ], "evidence_hash": "sha256:42ea83d2814e43721cc31dcf7b1f614e0a8da61f16e1ceba770717e53b9adc61", "state": "buildable" }, { "prerequisite_id": "semantic_state_grading", "checks": [ [ "independent_human_native_qualification_available", false ] ], "evidence_hash": "sha256:ffee53d223eefacf82059068ef42a5fd5244e601691d778decfa85057434d6ff", "state": "buildable" } ], "prerequisite_ids": [ "typed_tool_boundary", "semantic_state_grading" ], "component_hashes": [ [ "campaign_check", "sha256:cce99b5f9c7a0db014a7e214fd008ab609441e4e1530572bb96072d9f7786ad9" ], [ "manifest", "sha256:0d40666f55f47274554e2b4da2ad4a8f446797d0139b4d31f9f5afada82640e8" ], [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "population", "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b" ], [ "prerequisite:semantic_state_grading", "sha256:948c263f620779235b33b3e8180a40ad94906a09f84ea256ee69217f82e38b43" ], [ "prerequisite:typed_tool_boundary", "sha256:f5b00fea6435de713df0505ef831e0354e1a0e7662000ef7dcb6d6449fbbcb2f" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:9933e7b2fb4b7d6c382c5e2f35503cc53e8964a3f0fc469cae60ddff0e5bf643" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ], [ "test_receipt", "sha256:d1790dc366a92974e50650ad0f3a05f18872ddad86ac508b2a9ee208f136d88b" ] ], "issued_at": "2026-09-06T12:00:00+00:00", "valid_until": "2026-10-01T00:00:00Z", "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ], "campaign_check": { "issues": [], "status": "not_checked" }, "receipt_hash": "sha256:1134a0b24ec95cc14cb8d284f0d60778fd6cba22ce81a726a9a6448456190b91" }, "costs": { "agent_known_micro_usd": 1280000, "judge_known_micro_usd": 0, "known_subtotal_micro_usd": 1280000, "unknown_component_count": 16, "judge_not_dispatched_trials": 0, "complete_selected_estimate_micro_usd": null, "measurement_kind": "synthetic", "scope": "new agent/judge dispatch estimates in this packet; excludes prior attempts, tool/hosting/storage/human costs and invoice reconciliation", "deployment_authorized": false }, "candidate_joint_passes": 2, "candidate_structural_passes": 2, "candidate_completed_tasks": 3 }, { "candidate": "descriptive-control", "packet": { "manifest": { "experiment_id": "native-semantic-descriptive-control", "created_at": "2026-09-06T00:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "code_revision": "working-tree-unpinned", "model_id": "deterministic-controls-no-model", "prompt_version": "description-matching-control-v1", "tool_version": "multi-order-mock-v1", "dataset_version": "order-resolution-v1", "evaluator_version": "resolution-joint-v1", "policy_version": "resolution-contract-gate-v1", "environment_version": "python-3.12", "population_hash": "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b", "repetitions": 2, "measurement_kind": "synthetic", "estimand": "candidate_minus_baseline_resolution_and_qualified_truth", "statistical_method": "clustered_normal_interval", "non_inferiority_margin": 0.03, "confidence_level": 0.95, "minimum_independent_clusters": 30, "sequential_policy": "fixed_sample_no_interim_looks", "input_hashes": [ [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:be6dd99b15b00d998fa60d459366e7edc022f587a20d1a398c65dc8e593be5f7" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ] ], "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ] }, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "baseline_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:43fd9db6ac5a4c9ffcc4c0db888f9090cc8f461d76c3fe9d87bde3bf19dc9cdf" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:6c90d718cf3746ed0307399ebb9eef733cc0bd4f71ec52b6061de1b307e51003" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:01e5b777500db5936a0bc1c5d1bfefbf84a1f2f63221c772d6aa30bbd547887f" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:97b87740e8bb7dee10dc61a94771c8b16a6c875b6e856f6e00c1e2aed5ffdd6c" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:2e734f15c034610a551c69683462dd0019402f9335f612c988ee04ec4f0c7440" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:7cceb5cce9cd2765983cad2e7e3a0685fec7433546e386926dcd8beac1f180da" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:9369463a495d298079e66076b4d02fc328b82700e1344dbcfafc9e394b25bfac" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:dacc3639e9d98f86e6a0239ec1565cc4e34d19cd9d6ed3a1cf5106a8931bfb36" } ], "candidate_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:11ebfcae343e1966e90fd44b611aa0af436278ae2541f83023d11cc98b10efcc" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:768f0ea6055ce0372daeab38d20f4de6e055f31d43ef3d130a181eb3e06e9d0f" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:80839c7233b107417824f670955b6d05f0aad485494534105ea6586ffbb2f660" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:bbad775b80583f3eb89cfedba9319d5970571dc3b3adfa875542a6d6e378567e" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:ec68fda65c3ee01cbaccdb3af74bdf4e6d15092787d96adfe8b959d6d4bff86b" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:6bc7c006f2aed04241f574285ecc20ed4a6e0913adfed6fae5e43bc8da50a52b" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:0e1e38d11be5ec09d23194e4bbe45a9a79d1a0f6116d81c093ad15e15d680c7a" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "failed_checks": [], "artifact_hash": "sha256:2dffadf7454405d356ab0624f87fac76de3c7f6a9ab8f7e76ff18008bf2887a2" } ], "trial_artifacts": [ { "artifact_hash": "sha256:43fd9db6ac5a4c9ffcc4c0db888f9090cc8f461d76c3fe9d87bde3bf19dc9cdf", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06208301056176424, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:5403aec1cbeca5f40c9d2842430712087f0af28c1d3e81191d62708331cfa7bf", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:f4a1f72fa9548085a5eccaa95ae4a98b5740c2f570574be676488932b36dacf2", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:39d94636bcae9f5210f11fbbc48d14dddc335c1f4fd1879360c4ee81a7a547ec", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:11ebfcae343e1966e90fd44b611aa0af436278ae2541f83023d11cc98b10efcc", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.07095898035913706, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:5403aec1cbeca5f40c9d2842430712087f0af28c1d3e81191d62708331cfa7bf", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:7a4b30407e6dbd33ea28441c41d9952e419b4c246b5fe10ef17de92f4652cd96", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:39d94636bcae9f5210f11fbbc48d14dddc335c1f4fd1879360c4ee81a7a547ec", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:768f0ea6055ce0372daeab38d20f4de6e055f31d43ef3d130a181eb3e06e9d0f", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06445898907259107, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:b346cbd5ff6dd9c05d0bfdc5c145f13e0c89742315e5c0ab97ebaca1bdbbb26f", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:5b6383e7e66746adfcb732adadf6bd24d8568b75d6f3181acd2a5ca95a832def", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:605e2abaebf46f8d9d74c94925f4a4c2ba1ba125ae9a674b0d8a1c2515f64567", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:6c90d718cf3746ed0307399ebb9eef733cc0bd4f71ec52b6061de1b307e51003", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06204098463058472, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:b346cbd5ff6dd9c05d0bfdc5c145f13e0c89742315e5c0ab97ebaca1bdbbb26f", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:819505d4ae9d23e517ebcc519f593a414986b08f83150e29e17f262999cf31a7", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:605e2abaebf46f8d9d74c94925f4a4c2ba1ba125ae9a674b0d8a1c2515f64567", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:01e5b777500db5936a0bc1c5d1bfefbf84a1f2f63221c772d6aa30bbd547887f", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.0600829953327775, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:ba1e2ec18ee0e06a7063acc0d1ccecb5c43a5a8f6342baa1ac682fa05638c10c", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:2f4c8d79d53331b519cea2d4b2caf0df5318d0930c955c2c2d6fb6ed9830b0c8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:80839c7233b107417824f670955b6d05f0aad485494534105ea6586ffbb2f660", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06504199700430036, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:8d497b7b365e476443832b215c8659223a1209e6572899921a5e36c8df205357", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:79996d23bb01b642472d6152cfcb0d44d5dcd910476e13e5a9d09b7b5c6119b6", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:2f4c8d79d53331b519cea2d4b2caf0df5318d0930c955c2c2d6fb6ed9830b0c8", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:bbad775b80583f3eb89cfedba9319d5970571dc3b3adfa875542a6d6e378567e", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.032374984584748745, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:7486da165f48f4c91920ceb1ee003c934fffef3e31350988b49cd2580347999d", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:15e6b97614e6da85c5d9dc7d1ad97241325462f2f354c03e8eae3a09380aff58", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:786f4fb84cfd4915cb4afee2f3a283b1bb26f675d81fb2dc7e47f00fa8029a5f", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:97b87740e8bb7dee10dc61a94771c8b16a6c875b6e856f6e00c1e2aed5ffdd6c", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03183301305398345, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": false, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:7486da165f48f4c91920ceb1ee003c934fffef3e31350988b49cd2580347999d", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:e68a99c3bd40deb445c24cfb779654dfecbf46322535825e08186a379f35a784", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:786f4fb84cfd4915cb4afee2f3a283b1bb26f675d81fb2dc7e47f00fa8029a5f", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:ec68fda65c3ee01cbaccdb3af74bdf4e6d15092787d96adfe8b959d6d4bff86b", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06066699279472232, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:81a530912c8c61d7e4d2a8afbc90239f475d10e939027a448f5518d9737e7c92", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:191b6538d4b0c8b3856b6a532e684d7041ec0d9e3cf26c6e7afc38c76c14fabf", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:2e734f15c034610a551c69683462dd0019402f9335f612c988ee04ec4f0c7440", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06816699169576168, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:201786030448dee868bd9aed37974ed0b951fdf40a3637b6ac80b3235c651150", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:1a9e6eec6ec947f4636f360367e3258612c9712bb0bb46061a99b6fb9bbe726e", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:191b6538d4b0c8b3856b6a532e684d7041ec0d9e3cf26c6e7afc38c76c14fabf", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:7cceb5cce9cd2765983cad2e7e3a0685fec7433546e386926dcd8beac1f180da", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06204203236848116, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:adaa28561d53423afacf7e59e5a12e499bd7e3810f663fd4e29bdd28de32954b", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:4c9be6403f89458d2d74b57351fbcdbad44f31a60cda646764530f22cfc2dca8", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:8e28a7df9f648f4275b9b4bb09589feee54303eb2e80afbbae30e005b8ef4225", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:6bc7c006f2aed04241f574285ecc20ed4a6e0913adfed6fae5e43bc8da50a52b", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.06924994522705674, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:adaa28561d53423afacf7e59e5a12e499bd7e3810f663fd4e29bdd28de32954b", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:7ccac6508dcd529b496040c410470ddc2d3dd52a4669914493754c8e6b19c2a0", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:8e28a7df9f648f4275b9b4bb09589feee54303eb2e80afbbae30e005b8ef4225", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:0e1e38d11be5ec09d23194e4bbe45a9a79d1a0f6116d81c093ad15e15d680c7a", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.0631659640930593, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dac7707f547d1d2d08220d7d1bc59b50640842196f008b71e6e4ca9213671c09", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:c757282d234b37e604fec535aec4a6f848393749e4b1db82368832e0ad0b0619", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:00ab3621c547dbd1634a45eaa8b2d92187c73d333454bc7189d0468a876797ab", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:9369463a495d298079e66076b4d02fc328b82700e1344dbcfafc9e394b25bfac", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.059542013332247734, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": true, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:dac7707f547d1d2d08220d7d1bc59b50640842196f008b71e6e4ca9213671c09", "message_hash": "sha256:37373180816ec1fda80cb4c1c92f8e293fae8cfa7ba0cff7dfc55c81a4c10723", "passed": true, "structured_claim_hash": "sha256:d68a3396a0e4aa6e0a77275ca80db06250d2e717711a15f8cd3fd372462b7ae0" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:610aa64b10bcb18564a4e5f753bd34f87f9c7d8f763161f2f1390fd87d0b92c3", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ] }, "request_hash": "sha256:00ab3621c547dbd1634a45eaa8b2d92187c73d333454bc7189d0468a876797ab", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:dacc3639e9d98f86e6a0239ec1565cc4e34d19cd9d6ed3a1cf5106a8931bfb36", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.031333998776972294, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:65007e2e494d4f95913d1b13bfd81bedf5d81439bb0722806e7a061b8eee8fe6", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:75df134942818af56458ac36513e5358b1a1db9ad6a440275d850feaddb638d3", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:6d7630e3e231a94ca13b795155208f01cbbd99449b5284288c8620f5baf2e328", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:2dffadf7454405d356ab0624f87fac76de3c7f6a9ab8f7e76ff18008bf2887a2", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" } }, "elapsed_ms": 0.03162497887387872, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true }, { "name": "qualified_multi_order_message", "observed": "pass", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-joint-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_abstention_count": 0, "semantic_message_qualified": true, "semantic_status": { "abstained": false, "passed": true, "qualified": true, "reason": null }, "structural_contract_passed": true, "task_completed": false, "unqualified_message_count": 0 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [ "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b" ], "reverse": true, "schema": "resolution-trial-v2", "semantic_evaluation_receipt": { "abstained": false, "calibration_receipt_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "criterion_id": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "evidence_context_hash": "sha256:65007e2e494d4f95913d1b13bfd81bedf5d81439bb0722806e7a061b8eee8fe6", "message_hash": "sha256:9530f7257f1a096b7a551379cbe32c305e2e6df8f4addb5112c1511f05cf19fb", "passed": true, "structured_claim_hash": "sha256:242fc531ccc7c567c002ada4cdefdf817c4e674f66d50ec4fb49f6a4402384ce" }, "semantic_message_qualified": true, "semantic_stage": { "authority": "lab_only", "completed_at": "2026-09-06T12:00:00+00:00", "error_bounds": { "false_block_upper_95": 0.7763932022500211, "false_pass_upper_95": 0.7763932022500211 }, "invocation_id": "sha256:da2494ab9db3bdd385328d1b10a51e709699a08c93aea352008228c6a070fd26", "judge_latency_ms": 0, "judgment": { "campaign_audit_json": null, "explanation": "Synthetic literal rule, not general semantic validity; object intent is graded separately by the structural contract.", "provider_audit_json": null, "runtime_evidence": null, "verdict": "pass" }, "qualification": { "abstentions": 0, "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion_id": "multi_order_customer_message_truth_v1", "dataset_versions": [ "order-resolution-v1" ], "evaluator_version": "native-literal-fixture-v1", "evidence_kind": "synthetic", "expires_at": "2026-10-01T00:00:00Z", "false_blocks": 0, "false_examples": 2, "false_passes": 0, "issued_at": "2026-09-01T00:00:00Z", "label_artifact_hash": "sha256:c40ef716a7c067466446589bb7518b316c439cd48698c89af1696d6be2424977", "max_abstention_rate": 0.0, "max_false_block_upper": 0.8, "max_false_pass_upper": 0.8, "minimum_per_class": 2, "policy_versions": [ "multi-order-refund-policy-v1" ], "slice_scopes": [ [ "journey:refund", "surface:order-resolution" ] ], "truthful_examples": 2 }, "reason": null, "registration": { "allow_synthetic": true, "calibration_hash": "sha256:454a18e2f18782f37ecfb2c7832a259aead13455be9cacfe94a7464e0d54730b", "configuration_hash": "sha256:46e357cc131ca4e8ed0dfacb6862615990017be535f18e885f2d97f5ae0c854d", "criterion": "multi_order_customer_message_truth_v1", "evaluator_version": "native-literal-fixture-v1", "policy_version": "multi-order-refund-policy-v1" }, "request": { "customer_request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "execution_error": null, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "policy_version": "multi-order-refund-policy-v1", "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ] }, "request_hash": "sha256:6d7630e3e231a94ca13b795155208f01cbbd99449b5284288c8620f5baf2e328", "started_at": "2026-09-06T12:00:00+00:00", "status": "graded" }, "structural_contract_passed": true, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } } ] }, "test_evidence": { "checks": [ [ "all_registered_trials_replayed", true ] ], "packet_hash": "sha256:f0e8390160641f63341895293aa2c5b4cee1a3b17d3df8b233dc11e0cc635ab9" }, "release_receipt": { "experiment_id": "native-semantic-descriptive-control", "manifest_hash": "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3", "state": "locked", "action": "block", "authority_ceiling": "none", "raw_artifact_hash": "sha256:f0e8390160641f63341895293aa2c5b4cee1a3b17d3df8b233dc11e0cc635ab9", "deterministic_test_receipt": { "suite_id": "native-semantic-replay", "code_revision": "working-tree-unpinned", "checks": [ [ "all_registered_trials_replayed", true ] ], "artifact_hash": "sha256:c98184a5dc14e333f5ea00e96a6c7c1daf3867fd7df663097e05931ef8715649", "passed": true }, "deterministic_tests_passed": true, "hard_failure_count": 0, "comparison": { "method": "clustered_normal_interval", "confidence_level": 0.95, "margin": 0.03, "pair_count": 8, "independent_cluster_count": 1, "minimum_independent_clusters": 30, "baseline_rate": 1.0, "candidate_rate": 1.0, "point_difference": 0.0, "lower_bound": null, "upper_bound": null, "non_inferior": false, "superior": false, "status": "inconclusive" }, "prerequisite_receipts": [ { "prerequisite_id": "typed_tool_boundary", "checks": [ [ "full_boundary_qualification_available", false ] ], "evidence_hash": "sha256:42ea83d2814e43721cc31dcf7b1f614e0a8da61f16e1ceba770717e53b9adc61", "state": "buildable" }, { "prerequisite_id": "semantic_state_grading", "checks": [ [ "independent_human_native_qualification_available", false ] ], "evidence_hash": "sha256:ffee53d223eefacf82059068ef42a5fd5244e601691d778decfa85057434d6ff", "state": "buildable" } ], "prerequisite_ids": [ "typed_tool_boundary", "semantic_state_grading" ], "component_hashes": [ [ "campaign_check", "sha256:cce99b5f9c7a0db014a7e214fd008ab609441e4e1530572bb96072d9f7786ad9" ], [ "manifest", "sha256:11861a0a122a90a17248a5fd48f5ded03a23d53de2d2ba78a8cbe28c87342ce3" ], [ "native-semantic-registration", "sha256:8fac3621848dce0622971db0709e6e2dc07e362c2eaf4aedf36d1e3b341750f5" ], [ "population", "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b" ], [ "prerequisite:semantic_state_grading", "sha256:948c263f620779235b33b3e8180a40ad94906a09f84ea256ee69217f82e38b43" ], [ "prerequisite:typed_tool_boundary", "sha256:f5b00fea6435de713df0505ef831e0354e1a0e7662000ef7dcb6d6449fbbcb2f" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:be6dd99b15b00d998fa60d459366e7edc022f587a20d1a398c65dc8e593be5f7" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:a108afaf21c9d32aa94d29c65b6650755626fb21245b57bc80fc69f43f52cbb8" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:a01d8dce6a4ae26f35de1cadf668d3260c9db2b5ef5cc48a4bf6889478784850" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:dfbb2782ee82df16e61801d8d29e1fd5cf173ea43e731757f67fd3318d1ced91" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:1736ee37cde5603caaace131673e548d3d717d49947bfa1833fe75630169813f" ], [ "source:cx_eval_lab/resolution_semantic.py", "sha256:c0ccae209381c84dade3f62c80795d0240498ee37b105d2ea991b7c2ff6933e3" ], [ "source:cx_eval_lab/resolution_semantic_replay.py", "sha256:4e4ff20de89643a2bb018e5d5356b82c6cf6dc90aeea548385f5224f2b3f3b9c" ], [ "source:cx_eval_lab/resolution_semantic_study.py", "sha256:6ea89357f2d1d6a3abfa5e39c2148e97425436f2160dd460dc14a0fe929b9a88" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:bd62faa349f08959631c6e6e2c89a4ffa74ef49c9b09673d0e6020eadac595a5" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ], [ "test_receipt", "sha256:749c723bfc757d4d38ed2372c931cbd9ff0b684dd67a51814a6ada840e0222cd" ] ], "issued_at": "2026-09-06T12:00:00+00:00", "valid_until": "2026-10-01T00:00:00Z", "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ], "campaign_check": { "issues": [], "status": "not_checked" }, "receipt_hash": "sha256:7b9e4159fbd8ad7feac5554467b37ab4f2a60091879675396a8b163e43399919" }, "costs": { "agent_known_micro_usd": 1280000, "judge_known_micro_usd": 0, "known_subtotal_micro_usd": 1280000, "unknown_component_count": 16, "judge_not_dispatched_trials": 0, "complete_selected_estimate_micro_usd": null, "measurement_kind": "synthetic", "scope": "new agent/judge dispatch estimates in this packet; excludes prior attempts, tool/hosting/storage/human costs and invoice reconciliation", "deployment_authorized": false }, "candidate_joint_passes": 8, "candidate_structural_passes": 8, "candidate_completed_tasks": 6 } ], "revoked_assessment": { "packet_hash": "sha256:f0e8390160641f63341895293aa2c5b4cee1a3b17d3df8b233dc11e0cc635ab9", "registry_hash": "sha256:a494128e3dae349e2e455197bb7f4048325c4e3c35215dc62949e5fc8ed7f102", "historical_trust_hash": "sha256:39095c62296287965e477da20e4380e80897206a831ea75960c6a1b735c8d970", "synthetic_diagnostic": true, "checked_at": "2026-09-06T12:00:00+00:00", "replayed_trials": 16, "failed_trials": 0, "unqualified_message_trials": 0, "semantic_trials": 16, "calibration_status": "not_current", "issues": [ "sha256:43fd9db6ac5a4c9ffcc4c0db888f9090cc8f461d76c3fe9d87bde3bf19dc9cdf:qualification_missing_or_revoked", "sha256:11ebfcae343e1966e90fd44b611aa0af436278ae2541f83023d11cc98b10efcc:qualification_missing_or_revoked", "sha256:768f0ea6055ce0372daeab38d20f4de6e055f31d43ef3d130a181eb3e06e9d0f:qualification_missing_or_revoked", "sha256:6c90d718cf3746ed0307399ebb9eef733cc0bd4f71ec52b6061de1b307e51003:qualification_missing_or_revoked", "sha256:01e5b777500db5936a0bc1c5d1bfefbf84a1f2f63221c772d6aa30bbd547887f:qualification_missing_or_revoked", "sha256:80839c7233b107417824f670955b6d05f0aad485494534105ea6586ffbb2f660:qualification_missing_or_revoked", "sha256:bbad775b80583f3eb89cfedba9319d5970571dc3b3adfa875542a6d6e378567e:qualification_missing_or_revoked", "sha256:97b87740e8bb7dee10dc61a94771c8b16a6c875b6e856f6e00c1e2aed5ffdd6c:qualification_missing_or_revoked", "sha256:ec68fda65c3ee01cbaccdb3af74bdf4e6d15092787d96adfe8b959d6d4bff86b:qualification_missing_or_revoked", "sha256:2e734f15c034610a551c69683462dd0019402f9335f612c988ee04ec4f0c7440:qualification_missing_or_revoked", "sha256:7cceb5cce9cd2765983cad2e7e3a0685fec7433546e386926dcd8beac1f180da:qualification_missing_or_revoked", "sha256:6bc7c006f2aed04241f574285ecc20ed4a6e0913adfed6fae5e43bc8da50a52b:qualification_missing_or_revoked", "sha256:0e1e38d11be5ec09d23194e4bbe45a9a79d1a0f6116d81c093ad15e15d680c7a:qualification_missing_or_revoked", "sha256:9369463a495d298079e66076b4d02fc328b82700e1344dbcfafc9e394b25bfac:qualification_missing_or_revoked", "sha256:dacc3639e9d98f86e6a0239ec1565cc4e34d19cd9d6ed3a1cf5106a8931bfb36:qualification_missing_or_revoked", "sha256:2dffadf7454405d356ab0624f87fac76de3c7f6a9ab8f7e76ff18008bf2887a2:qualification_missing_or_revoked" ], "deployment_authorized": false }, "limitations": "Four synthetic rows and synthetic reviewers, not independent human labels. Fixture group IDs identify executions, not independent customers: all cases share one customer. Calibration overlaps comparison controls; no held-out evaluation exists. The permissive n=2 and 0.80 upper-bound policy exercises wiring only. The literal judge recognizes three sentences and does not infer arbitrary customer intent. Two permutations are not independent samples. Agent cost/latency profiles are synthetic; mock wall times are retained separately. No live models, authenticated review, general semantic validity, or deployment qualification." }