{ "study": "paired-order-resolution-study-v1", "evidence_kind": "executed_deterministic_mock", "deployment_authorized": false, "comparisons": [ { "candidate": "first-record-mutant", "packet": { "manifest": { "experiment_id": "paired-resolution-first-record-mutant", "created_at": "2026-09-06T00:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "code_revision": "working-tree-unpinned", "model_id": "deterministic-controls-no-model", "prompt_version": "description-matching-control-v1", "tool_version": "multi-order-mock-v1", "dataset_version": "order-resolution-v1", "evaluator_version": "resolution-contract-v1", "policy_version": "resolution-contract-gate-v1", "environment_version": "python-3.12", "population_hash": "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b", "repetitions": 2, "measurement_kind": "synthetic", "estimand": "candidate_minus_baseline_resolution_contract", "statistical_method": "clustered_normal_interval", "non_inferiority_margin": 0.03, "confidence_level": 0.95, "minimum_independent_clusters": 30, "sequential_policy": "fixed_sample_no_interim_looks", "input_hashes": [ [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:a53da74a881d42e4bb1c9f046a7e7d8f22193e7f81cdc0e695e54d579022d7f0" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:7dd841f534037962e42701ada62d0e89755f8d71876867c8289a150d911b2684" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:e51886e1346309716523b5c3e44d86c5e008109a1c1e534472eee11855321b03" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:cb9b940e780dc97cd42a2da1eac4dab18f4db2ed8d6700ab275aa443206726b4" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:81fe4c05d8fa746e943a6ca32f0d95653ebaa1d71a138c25ac425dbadd82b8da" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:6299a0863bae84ea87b8901b846f2e1563e2bcb60c75be378d42f615e3f5328e" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ] ], "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ] }, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "baseline_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:d10b1d4babf4b0dceec6ad7f5d30412dc9fb2349924ef11ef52644d6222326f7" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:5d19759e7881a0ceef20f5334974cb6f0840b271687b37c6b2acdbec3a2f477b" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:51b6e47b703ba7af3cfff25d079846b3606fe24ed0500e2fe11e5d9289d2579b" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:711ed464b04f15c8899960b824b6649bb7dc8848eccfa638778ead16b43e4407" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:38a8f0db9de48e58f3751f2c4a26da95dc0c623707f1a553b54ab46e077aa331" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:c40234e288f33b1ba832889dbfc0c1dcdbc438358c75c8941107323b942d8e92" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:8ef3ddde0ea712ca9afcd8692e0c82df0fff308fc6373223c00dc6cf0538e61e" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:c8c41d288a6b8abc031e09b15100f45b33895b96fe82f9e86c638e9f9639f432" } ], "candidate_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "wrong_order_commits", "resolution_contract" ], "artifact_hash": "sha256:bc5fa24f4b24eaca9b4725f224b788e20762bfe3a690c3b5d2728cd3c282e26f" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:bd9d36007b55a726afafecf7f158d91bb9540bebf18908801c81887e974e8fe9" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:14f0bdb99c7e6f943ae505b12d4ee5a649495962aeeeb6953880c601dedcc9e2" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:1a6dcb98c836dc0bdc80d05ff5481491f7fa4cf750cef1b03b938407d158bdcb" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [], "artifact_hash": "sha256:89d6de48496e1122ecdf354dcf8badba625a45124b63eae19d9f2d052566b23c" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:88a98599f1f447bf569581e9a724f62d4152c137c9aafbdebd42846c133a4e47" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "wrong_order_commits", "resolution_contract" ], "artifact_hash": "sha256:5a0ef25ee56ca069f0f2642468ff52c031acb2ea12b86d3112175a51847d13d7" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": false, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "failed_checks": [ "wrong_order_commits", "missing_clarifications", "premature_action_attempts", "resolution_contract" ], "artifact_hash": "sha256:6009a949a2e2f74b99d85eaad50ffcf95926ec0d0766aeb8e441d1d0b79e2c9a" } ], "trial_artifacts": [ { "artifact_hash": "sha256:d10b1d4babf4b0dceec6ad7f5d30412dc9fb2349924ef11ef52644d6222326f7", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.10608299635350704, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:bc5fa24f4b24eaca9b4725f224b788e20762bfe3a690c3b5d2728cd3c282e26f", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.07324997568503022, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": false, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:bd9d36007b55a726afafecf7f158d91bb9540bebf18908801c81887e974e8fe9", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06533402483910322, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": false, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:5d19759e7881a0ceef20f5334974cb6f0840b271687b37c6b2acdbec3a2f477b", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.07237499812617898, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:51b6e47b703ba7af3cfff25d079846b3606fe24ed0500e2fe11e5d9289d2579b", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06758398376405239, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:14f0bdb99c7e6f943ae505b12d4ee5a649495962aeeeb6953880c601dedcc9e2", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05829200381413102, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:1a6dcb98c836dc0bdc80d05ff5481491f7fa4cf750cef1b03b938407d158bdcb", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05966698518022895, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:711ed464b04f15c8899960b824b6649bb7dc8848eccfa638778ead16b43e4407", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.040834012906998396, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:89d6de48496e1122ecdf354dcf8badba625a45124b63eae19d9f2d052566b23c", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06733299233019352, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:38a8f0db9de48e58f3751f2c4a26da95dc0c623707f1a553b54ab46e077aa331", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.058917037677019835, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c40234e288f33b1ba832889dbfc0c1dcdbc438358c75c8941107323b942d8e92", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06033299723640084, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:88a98599f1f447bf569581e9a724f62d4152c137c9aafbdebd42846c133a4e47", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05633296677842736, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:5a0ef25ee56ca069f0f2642468ff52c031acb2ea12b86d3112175a51847d13d7", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.056999968364834785, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": false, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } }, { "artifact_hash": "sha256:8ef3ddde0ea712ca9afcd8692e0c82df0fff308fc6373223c00dc6cf0538e61e", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06333395140245557, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c8c41d288a6b8abc031e09b15100f45b33895b96fe82f9e86c638e9f9639f432", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.03445800393819809, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:6009a949a2e2f74b99d85eaad50ffcf95926ec0d0766aeb8e441d1d0b79e2c9a", "payload": { "agent": "first-record-mutant", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "first-record-mutant", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05724997026845813, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "1", "passed": false }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "1", "passed": false }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "1", "passed": false }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": false } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 1, "passed": false, "premature_action_attempts": 1, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 1 }, "passed": false, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 1, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": false, "premature_action_attempts": 1, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 1 } } ] }, "test_evidence": { "checks": [ [ "all_registered_trials_replayed", true ] ], "packet_hash": "sha256:660c8855149eae0b7f1bf6be19f4420a1cbfd5b523c50f41714ab7536c3c9442" }, "release_receipt": { "experiment_id": "paired-resolution-first-record-mutant", "manifest_hash": "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e", "state": "locked", "action": "block", "authority_ceiling": "none", "raw_artifact_hash": "sha256:660c8855149eae0b7f1bf6be19f4420a1cbfd5b523c50f41714ab7536c3c9442", "deterministic_test_receipt": { "suite_id": "resolution-mock-replay", "code_revision": "working-tree-unpinned", "checks": [ [ "all_registered_trials_replayed", true ] ], "artifact_hash": "sha256:d7b1aa2a47ca0b0b289d499c2b6741d8ed90e85a59464db2764e88c25236e0d1", "passed": true }, "deterministic_tests_passed": true, "hard_failure_count": 6, "comparison": { "method": "clustered_normal_interval", "confidence_level": 0.95, "margin": 0.03, "pair_count": 8, "independent_cluster_count": 1, "minimum_independent_clusters": 30, "baseline_rate": 1.0, "candidate_rate": 0.25, "point_difference": -0.75, "lower_bound": null, "upper_bound": null, "non_inferior": false, "superior": false, "status": "inconclusive" }, "prerequisite_receipts": [ { "prerequisite_id": "typed_tool_boundary", "checks": [ [ "full_boundary_qualification_available", false ] ], "evidence_hash": "sha256:42ea83d2814e43721cc31dcf7b1f614e0a8da61f16e1ceba770717e53b9adc61", "state": "buildable" }, { "prerequisite_id": "semantic_state_grading", "checks": [ [ "multi_order_semantic_qualification_available", false ] ], "evidence_hash": "sha256:948fd4a7b768857a7958e5cbb9e59c935efaf5e06109f03b02f369feb8d089c9", "state": "buildable" } ], "prerequisite_ids": [ "typed_tool_boundary", "semantic_state_grading" ], "component_hashes": [ [ "campaign_check", "sha256:cce99b5f9c7a0db014a7e214fd008ab609441e4e1530572bb96072d9f7786ad9" ], [ "manifest", "sha256:ff118dd4483ae38ddf3af42ffc2f0ad73854ea70c319e060e755fcf56c716e9e" ], [ "population", "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b" ], [ "prerequisite:semantic_state_grading", "sha256:7551759f027544c6b0baf58bc98f5b90207e6009ffce9ac6b537bba489f879f9" ], [ "prerequisite:typed_tool_boundary", "sha256:f5b00fea6435de713df0505ef831e0354e1a0e7662000ef7dcb6d6449fbbcb2f" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:a53da74a881d42e4bb1c9f046a7e7d8f22193e7f81cdc0e695e54d579022d7f0" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:7dd841f534037962e42701ada62d0e89755f8d71876867c8289a150d911b2684" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:e51886e1346309716523b5c3e44d86c5e008109a1c1e534472eee11855321b03" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:cb9b940e780dc97cd42a2da1eac4dab18f4db2ed8d6700ab275aa443206726b4" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:81fe4c05d8fa746e943a6ca32f0d95653ebaa1d71a138c25ac425dbadd82b8da" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:6299a0863bae84ea87b8901b846f2e1563e2bcb60c75be378d42f615e3f5328e" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ], [ "test_receipt", "sha256:85c5cc8d40d5f4fd245173c01f8852e4cb515aa20fb81bba36371ac74e949b50" ] ], "issued_at": "2026-09-06T12:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ], "campaign_check": { "issues": [], "status": "not_checked" }, "receipt_hash": "sha256:57cfac917c5bfda4437d37871a3c516c28250273f4a95a94a78e5e452639c969" }, "costs": { "agent_known_micro_usd": 1280000, "judge_known_micro_usd": 0, "known_subtotal_micro_usd": 1280000, "unknown_component_count": 0, "judge_not_dispatched_trials": 16, "complete_selected_estimate_micro_usd": 1280000, "measurement_kind": "synthetic", "scope": "new agent/judge dispatch estimates in this packet; excludes prior attempts, tool/hosting/storage/human costs and invoice reconciliation", "deployment_authorized": false }, "candidate_contract_passes": 2, "candidate_completed_tasks": 3, "unqualified_message_trials": 16 }, { "candidate": "descriptive-control", "packet": { "manifest": { "experiment_id": "paired-resolution-descriptive-control", "created_at": "2026-09-06T00:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "code_revision": "working-tree-unpinned", "model_id": "deterministic-controls-no-model", "prompt_version": "description-matching-control-v1", "tool_version": "multi-order-mock-v1", "dataset_version": "order-resolution-v1", "evaluator_version": "resolution-contract-v1", "policy_version": "resolution-contract-gate-v1", "environment_version": "python-3.12", "population_hash": "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b", "repetitions": 2, "measurement_kind": "synthetic", "estimand": "candidate_minus_baseline_resolution_contract", "statistical_method": "clustered_normal_interval", "non_inferiority_margin": 0.03, "confidence_level": 0.95, "minimum_independent_clusters": 30, "sequential_policy": "fixed_sample_no_interim_looks", "input_hashes": [ [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:919024429a8cc710c3fb6a6158fd6efd3fdd0f208b5bd811e15e60aa11b86833" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:7dd841f534037962e42701ada62d0e89755f8d71876867c8289a150d911b2684" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:e51886e1346309716523b5c3e44d86c5e008109a1c1e534472eee11855321b03" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:cb9b940e780dc97cd42a2da1eac4dab18f4db2ed8d6700ab275aa443206726b4" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:81fe4c05d8fa746e943a6ca32f0d95653ebaa1d71a138c25ac425dbadd82b8da" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:6299a0863bae84ea87b8901b846f2e1563e2bcb60c75be378d42f615e3f5328e" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ] ], "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ] }, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "baseline_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:7ced54a8cda1f2dea7616ee35fcf3f9b717a43f63265d732e6656471b12a2214" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:b7a47ac4959d3b7c66629097a6d3ecd0961f04aedf627d7ed7e8d09b62697304" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:c1c8444fb9337a73f7b5d382b339fc399e2372c3939a79aa2b2a1e2c303c74fa" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:f110e782899a2be1e1d5baac26b65b3e3a7078fb0430c92d7b0988cc7d9e6a04" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:3bfe8d6048474b07930bd2da95ab05d2fb0171ce1c69afb73270297eb8cbdd58" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:9542d5cfdfbce9472997ab1b151fbcfc051c8a4e9a76dad95bc8d5052300e042" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:7f493db2f370710b6a830585e7e26996003eb2973510e561a42b8acfab749653" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "baseline", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:16dcbdb2a6bbee1e493928ae40d0d7b36432c7e70346e8b1a2d3e3179ee1b15b" } ], "candidate_trials": [ { "case_id": "explicit-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:07c4001b92f0d8d507ad2242f2c69cc6ea491a64e2272aa072ffc050087a8d11" }, { "case_id": "ambiguous-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:e9700138b422e8d49364917099e9f58021913983d9698c44cccd21148514b753" }, { "case_id": "customer-correction", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:e29861da6f1efd5fa6ad29ded48c3ff982fa17dc7299d4b1a22051b4ec4a7d1f" }, { "case_id": "unresolved-description", "trial_index": 0, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:43ae088e32b4480b3366bff1f85e3bf83c8022069ac0541e46f9b3c0ec336907" }, { "case_id": "explicit-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:4306486f8f927fa168f1ad0243f462a876e2c0297f1e7a3e154d0149277adae3" }, { "case_id": "ambiguous-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:98f4ed0fc66079922701971811252fe6290ec997581119feee09911436f20ba2" }, { "case_id": "customer-correction", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:fb9b549d55f4e1b9c09e5005fca11e6d7fb63852af92ff1afac700541fad6a9d" }, { "case_id": "unresolved-description", "trial_index": 1, "cluster_id": "customer-1", "arm": "candidate", "passed": true, "latency_ms": 250, "cost_usd": 0.08, "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "failed_checks": [], "artifact_hash": "sha256:067fb2ed70b194977220064e8347bfcce247b639bc79e89c0cdbae5e8ea74c86" } ], "trial_artifacts": [ { "artifact_hash": "sha256:7ced54a8cda1f2dea7616ee35fcf3f9b717a43f63265d732e6656471b12a2214", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.07295800605788827, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:07c4001b92f0d8d507ad2242f2c69cc6ea491a64e2272aa072ffc050087a8d11", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06074999691918492, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:e9700138b422e8d49364917099e9f58021913983d9698c44cccd21148514b753", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06233301246538758, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:b7a47ac4959d3b7c66629097a6d3ecd0961f04aedf627d7ed7e8d09b62697304", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06074999691918492, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:c1c8444fb9337a73f7b5d382b339fc399e2372c3939a79aa2b2a1e2c303c74fa", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05629099905490875, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:e29861da6f1efd5fa6ad29ded48c3ff982fa17dc7299d4b1a22051b4ec4a7d1f", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.056416960433125496, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:43ae088e32b4480b3366bff1f85e3bf83c8022069ac0541e46f9b3c0ec336907", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.03108399687334895, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:f110e782899a2be1e1d5baac26b65b3e3a7078fb0430c92d7b0988cc7d9e6a04", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.029125018045306206, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 0 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": false, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" }, { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:4306486f8f927fa168f1ad0243f462a876e2c0297f1e7a3e154d0149277adae3", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05858304211869836, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "explicit-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:3bfe8d6048474b07930bd2da95ab05d2fb0171ce1c69afb73270297eb8cbdd58", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "authority": "lab_only", "case": { "case_id": "explicit-description", "clarification_reply": null, "expected_order_id": "order-b", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06112497067078948, "evaluation": { "case_id": "explicit-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "explicit-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-b" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-b", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-b", "refund_idempotency_keys": [ "refund:order-b" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-b" } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-b" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-b" ], "result": { "amount_cents": 4500, "currency": "EUR", "order_id": "order-b" }, "tool": "get_order" }, { "arguments": [ "order-b" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-b", 4500, "EUR", null, "refund:order-b" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:9542d5cfdfbce9472997ab1b151fbcfc051c8a4e9a76dad95bc8d5052300e042", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.06070802919566631, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "ambiguous-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:98f4ed0fc66079922701971811252fe6290ec997581119feee09911436f20ba2", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "ambiguous-description", "clarification_reply": "The blue backpack bought on 2026-08-01.", "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.0600829953327775, "evaluation": { "case_id": "ambiguous-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "ambiguous-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": "The blue backpack bought on 2026-08-01.", "tool": "ask_customer" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:fb9b549d55f4e1b9c09e5005fca11e6d7fb63852af92ff1afac700541fad6a9d", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05683297058567405, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "customer-correction", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:7f493db2f370710b6a830585e7e26996003eb2973510e561a42b8acfab749653", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "authority": "lab_only", "case": { "case_id": "customer-correction", "clarification_reply": null, "expected_order_id": "order-a", "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund the blue backpack bought on 2026-08-02. Correction: the blue backpack bought on 2026-08-01." }, "required_clarifications": 0 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.05600001895800233, "evaluation": { "case_id": "customer-correction", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": true, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": true, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "customer-correction", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [ { "details": { "customer_id": "customer-1", "order_id": "order-a" }, "sequence": 1, "status": "verified", "tool": "verify_identity" }, { "details": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "sequence": 2, "status": "success", "tool": "get_order" }, { "details": { "approval_threshold_cents": 10000, "eligible": true }, "sequence": 3, "status": "success", "tool": "consult_refund_policy" }, { "details": { "approval_granted": false, "eligible": true, "idempotency_key": "refund:order-a", "identity_verified": true, "policy_consulted": true }, "sequence": 4, "status": "committed", "tool": "issue_refund" } ], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": true, "policy_consulted": true, "policy_order_id": "order-a", "refund_idempotency_keys": [ "refund:order-a" ], "refund_transaction_count": 1, "timeout_delivered": false, "verified_customer_id": "customer-1", "verified_order_id": "order-a" } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "refunded", "escalation_reason": null, "message": "Your refund has been confirmed.", "message_template_id": "refund_committed_v1", "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "committed" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": true, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "customer-1", "order-a" ], "result": true, "tool": "verify_identity" }, { "arguments": [ "order-a" ], "result": { "amount_cents": 4000, "currency": "USD", "order_id": "order-a" }, "tool": "get_order" }, { "arguments": [ "order-a" ], "result": { "approval_threshold_cents": 10000, "eligible": true }, "tool": "consult_refund_policy" }, { "arguments": [ "order-a", 4000, "USD", null, "refund:order-a" ], "result": { "status": "committed" }, "tool": "issue_refund" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:16dcbdb2a6bbee1e493928ae40d0d7b36432c7e70346e8b1a2d3e3179ee1b15b", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.030916999094188213, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "baseline", "case_id": "unresolved-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } }, { "artifact_hash": "sha256:067fb2ed70b194977220064e8347bfcce247b639bc79e89c0cdbae5e8ea74c86", "payload": { "agent": "descriptive-control", "agent_input": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "authority": "lab_only", "case": { "case_id": "unresolved-description", "clarification_reply": null, "expected_order_id": null, "orders": [ { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" } ], "request": { "customer_id": "customer-1", "utterance": "Refund my blue backpack." }, "required_clarifications": 1 }, "cost_usd": 0.08, "dataset_version": "order-resolution-v1", "denied_attempts": 0, "design": { "arm_order": "alternate_by_case_and_repetition", "baseline_agent": "descriptive-control", "candidate_agent": "descriptive-control", "criterion": "resolution-contract-v1", "order_permutation": "reverse_on_odd_repetitions", "semantic_qualification": "not_implemented" }, "elapsed_ms": 0.028499984182417393, "evaluation": { "case_id": "unresolved-description", "checks": [ { "name": "wrong_order_commits", "observed": "0", "passed": true }, { "name": "unnecessary_clarifications", "observed": "0", "passed": true }, { "name": "missing_clarifications", "observed": "0", "passed": true }, { "name": "denied_attempts", "observed": "0", "passed": true }, { "name": "premature_action_attempts", "observed": "0", "passed": true }, { "name": "resolution_contract", "observed": "structural outcome and response enum", "passed": true } ], "cost_usd": 0.08, "criterion": "resolution-contract-v1", "latency_ms": 250, "metrics": { "denied_attempts": 0, "missing_clarifications": 0, "passed": true, "premature_action_attempts": 0, "task_completed": false, "unnecessary_clarifications": 0, "wrong_order_commits": 0 }, "passed": true, "semantic_message_qualified": false, "task_completed": false, "unqualified_message_count": 1 }, "execution_error": null, "identity": { "arm": "candidate", "case_id": "unresolved-description", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "trial_index": 1 }, "latency_ms": 250, "measurement": { "cost_usd_per_case": 0.08, "evidence_kind": "synthetic", "latency_ms": 250, "source": "harness-owned teaching profile" }, "missing_clarifications": 0, "orders": [ { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-other", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-c" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4500, "currency": "EUR", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-02", "eligible": true, "order_id": "order-b" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } }, { "events": [], "order": { "amount_cents": 4000, "currency": "USD", "customer_id": "customer-1", "description": "blue backpack bought on 2026-08-01", "eligible": true, "order_id": "order-a" }, "state": { "approval_granted": false, "approval_id": null, "approval_order_id": null, "approved_amount_cents": null, "approved_currency": null, "identity_verified": false, "policy_consulted": false, "policy_order_id": null, "refund_idempotency_keys": [], "refund_transaction_count": 0, "timeout_delivered": false, "verified_customer_id": null, "verified_order_id": null } } ], "output": { "arrival_commitment_days": null, "claimed_outcome": "needs_review", "escalation_reason": null, "message": "Please clarify which purchase you mean.", "message_template_id": null, "runtime_evidence": null, "settlement_status_claim": "not_claimed", "transaction_status_claim": "unknown" }, "passed": true, "premature_action_attempts": 0, "qualified_semantic_calibration_hashes": [], "reverse": true, "schema": "resolution-trial-v1", "semantic_evaluation_receipt": null, "semantic_message_qualified": false, "semantic_stage": null, "task_completed": false, "tool_events": [ { "arguments": [ "customer-1" ], "result": [ { "description": "blue backpack bought on 2026-08-02", "order_id": "order-b" }, { "description": "blue backpack bought on 2026-08-01", "order_id": "order-a" } ], "tool": "list_orders" }, { "arguments": [ "Which item and purchase date do you mean?" ], "result": null, "tool": "ask_customer" } ], "unnecessary_clarifications": 0, "wrong_order_commits": 0 } } ] }, "test_evidence": { "checks": [ [ "all_registered_trials_replayed", true ] ], "packet_hash": "sha256:b196a4febe9acb2495e0391f5db983af1c3e511a5a7c7f375dcf2025e5d7adf2" }, "release_receipt": { "experiment_id": "paired-resolution-descriptive-control", "manifest_hash": "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f", "state": "locked", "action": "block", "authority_ceiling": "none", "raw_artifact_hash": "sha256:b196a4febe9acb2495e0391f5db983af1c3e511a5a7c7f375dcf2025e5d7adf2", "deterministic_test_receipt": { "suite_id": "resolution-mock-replay", "code_revision": "working-tree-unpinned", "checks": [ [ "all_registered_trials_replayed", true ] ], "artifact_hash": "sha256:9bcf2172a0a691401c099f9b7ba6a3495ecab913cd21b4b5b1c33c8246a1777b", "passed": true }, "deterministic_tests_passed": true, "hard_failure_count": 0, "comparison": { "method": "clustered_normal_interval", "confidence_level": 0.95, "margin": 0.03, "pair_count": 8, "independent_cluster_count": 1, "minimum_independent_clusters": 30, "baseline_rate": 1.0, "candidate_rate": 1.0, "point_difference": 0.0, "lower_bound": null, "upper_bound": null, "non_inferior": false, "superior": false, "status": "inconclusive" }, "prerequisite_receipts": [ { "prerequisite_id": "typed_tool_boundary", "checks": [ [ "full_boundary_qualification_available", false ] ], "evidence_hash": "sha256:42ea83d2814e43721cc31dcf7b1f614e0a8da61f16e1ceba770717e53b9adc61", "state": "buildable" }, { "prerequisite_id": "semantic_state_grading", "checks": [ [ "multi_order_semantic_qualification_available", false ] ], "evidence_hash": "sha256:948fd4a7b768857a7958e5cbb9e59c935efaf5e06109f03b02f369feb8d089c9", "state": "buildable" } ], "prerequisite_ids": [ "typed_tool_boundary", "semantic_state_grading" ], "component_hashes": [ [ "campaign_check", "sha256:cce99b5f9c7a0db014a7e214fd008ab609441e4e1530572bb96072d9f7786ad9" ], [ "manifest", "sha256:3eeb00c97ef6a3ec3f3bd07d5c91f40b82be9b6cf7d787977cf52f142a54308f" ], [ "population", "sha256:4427606a697a603e5713fa7af0ec18de8265de1c7c86a5d08bf55791e0f99e3b" ], [ "prerequisite:semantic_state_grading", "sha256:7551759f027544c6b0baf58bc98f5b90207e6009ffce9ac6b537bba489f879f9" ], [ "prerequisite:typed_tool_boundary", "sha256:f5b00fea6435de713df0505ef831e0354e1a0e7662000ef7dcb6d6449fbbcb2f" ], [ "resolution-cases", "sha256:60ac8879bee8ac0e14409bf2e448a01021205850e7761c8b1b14d5901631d739" ], [ "resolution-design", "sha256:919024429a8cc710c3fb6a6158fd6efd3fdd0f208b5bd811e15e60aa11b86833" ], [ "source:cx_eval_lab/__init__.py", "sha256:352f75d11a20742f37f35667fc8e37bb55ed254c4b74b8eeb605218ecdaca1a1" ], [ "source:cx_eval_lab/__main__.py", "sha256:75c2e5b562a49de5d3eed8e8886c23863ac8059aefbf27eb14bf97feef02856e" ], [ "source:cx_eval_lab/advanced.py", "sha256:d4656eded62f9ba2aa7238e897eb3e6b0be9d58c7ca4efe68c8cacd7a60e417b" ], [ "source:cx_eval_lab/agents.py", "sha256:80cd378bb1e3f2739cf931d49bdbe369b19fa688c98490801f322237aa2e3c02" ], [ "source:cx_eval_lab/artifacts.py", "sha256:7dd841f534037962e42701ada62d0e89755f8d71876867c8289a150d911b2684" ], [ "source:cx_eval_lab/budgeted_judge.py", "sha256:b30d4af0bd32f8943a6084157603e1d022b3616aca0ed9df745840dd8a411a8a" ], [ "source:cx_eval_lab/calibration_data.py", "sha256:0568d843fd428d435b68cf422dd683b70297ce8296b7796bf0537eae6a0f59fa" ], [ "source:cx_eval_lab/campaign_budget.py", "sha256:38dd2f28216fae871eabff7e324061a1b4c1f78ba869da285cb06c1aaa07e815" ], [ "source:cx_eval_lab/campaign_conformance.py", "sha256:7e0198275eef4eb404a5f2c83e2af70f81477996b7bd5ab75ec3995c4748d8a4" ], [ "source:cx_eval_lab/campaign_gate.py", "sha256:b0935bc913bd26f096dca063bd1240ca6ed802979a4ec4d36ac3ab3e6341e209" ], [ "source:cx_eval_lab/campaign_study.py", "sha256:fa8715567fd5bd511a5e381e51c59f0d7637129a2afd5aec469067e4e66dfc8a" ], [ "source:cx_eval_lab/ci_conformance.py", "sha256:bf888a2238a633e315ae11fca63e9dd9e038c0f17fdaac562808a8169f609b93" ], [ "source:cx_eval_lab/cost_accounting.py", "sha256:c8831d6fde32d72c3ff3b9ca6befdac751b5a3de9064670da4bbad0407565091" ], [ "source:cx_eval_lab/dataset.py", "sha256:d5f4a0cbf77f02c40dd6976d0c51e144d7c7020d8ada2e6bc8f12a615e0b9830" ], [ "source:cx_eval_lab/evaluators.py", "sha256:232bfa08db8ed11bad6df3154ed4cac79d8f7ce29bc4bba58819fe1b5c073cb1" ], [ "source:cx_eval_lab/evidence.py", "sha256:d60bf77ec43f037e6c30bf24a3d57d0a82b67f49deec4ddb321618453f0b256b" ], [ "source:cx_eval_lab/exposure_control.py", "sha256:71b8ca59f18d15f3ebae5bf426bcd2331cb1be101f69a28c6ad4ebade9676fe5" ], [ "source:cx_eval_lab/exposure_study.py", "sha256:471924d2082bc270e6799429d6179ce040cc84fe25cfb56d6c6d03dd66fbe609" ], [ "source:cx_eval_lab/gate.py", "sha256:4fe0d5538588c83f415f3fc6dca8e9f432c33a256ebddb32bd3055900665ec83" ], [ "source:cx_eval_lab/models.py", "sha256:a4fdac04db9cf505819cb1fbeed553506ec74d194055ba12f710968f3785f895" ], [ "source:cx_eval_lab/openai_judge.py", "sha256:a9bcf76704c98f1876ea35d5a1119d1ec7e0d63291b7cbe08c467443b3cf57ec" ], [ "source:cx_eval_lab/openai_runtime.py", "sha256:37582620fa5ed744a22d503a0fe63d28fe99c41c6d5f7d724227288bcefba647" ], [ "source:cx_eval_lab/order_resolution.py", "sha256:1540351815ac842a506660c1f3fdee1c8a5575fb6f9386814121c431d970f33f" ], [ "source:cx_eval_lab/recovery_study.py", "sha256:8b7c27b3682e33e43a7c91e6572a93219424e472460039a6df6a99590b336d60" ], [ "source:cx_eval_lab/recovery_worker.py", "sha256:87a4da856afdf16a9a5a124699dc62994df2575c7622584fdbd8f88435af3547" ], [ "source:cx_eval_lab/replay_authority.py", "sha256:e51886e1346309716523b5c3e44d86c5e008109a1c1e534472eee11855321b03" ], [ "source:cx_eval_lab/resolution_evidence.py", "sha256:cb9b940e780dc97cd42a2da1eac4dab18f4db2ed8d6700ab275aa443206726b4" ], [ "source:cx_eval_lab/resolution_paired_study.py", "sha256:885fd703fdac70c3ca85b372fcdac0642628ee33127dd032094c344ba0172725" ], [ "source:cx_eval_lab/resolution_runner.py", "sha256:81fe4c05d8fa746e943a6ca32f0d95653ebaa1d71a138c25ac425dbadd82b8da" ], [ "source:cx_eval_lab/runner.py", "sha256:d29a5683a50c9f1afd8a414883d2c1f1ef5154fa86788ae9dba8fc3da29186be" ], [ "source:cx_eval_lab/semantic.py", "sha256:6299a0863bae84ea87b8901b846f2e1563e2bcb60c75be378d42f615e3f5328e" ], [ "source:cx_eval_lab/sequential_study.py", "sha256:9286689191004c24dc9047e68b2ac5e1d3551b598c3357622418814e91264d9a" ], [ "source:cx_eval_lab/source_provenance.py", "sha256:4fdfe701884aa8ebc919509e97e11e12a0387d78328424ea0be22157823407ed" ], [ "source:cx_eval_lab/statistical_study.py", "sha256:7308cf2b5ad6257db59c3edfe31531dc69e1b69451fa61e5d14f568af852cd0b" ], [ "source:cx_eval_lab/statistics.py", "sha256:1568c8046ef1c57b74cf2535e14c104b380c56e20af812e94bfa81d6b849305b" ], [ "source:cx_eval_lab/world.py", "sha256:086922f61fd9c4fb3f4c267a94153ba6b50d0b21486fa6d0c2cbb5595263a3cd" ], [ "source:pyproject.toml", "sha256:2f2fd662fa150fc9b09e2578645ef4c61af75e9278dbb651e886a62e96b0bffc" ], [ "source:uv.lock", "sha256:6c9ba06f8dfaaf440f6bab0a31da525baee10c330c984049e9642eb875df6ee8" ], [ "test_receipt", "sha256:4320f900228085c4d2f2c41de5bb9b1957b1806e3ac5eba8b7e3e408e6c6449d" ] ], "issued_at": "2026-09-06T12:00:00Z", "valid_until": "2026-10-01T00:00:00Z", "invalidation_rules": [ "case_or_design_change", "source_change", "semantic_scope_not_qualified" ], "campaign_check": { "issues": [], "status": "not_checked" }, "receipt_hash": "sha256:18a37393d085fb327662657fbb19b745c230cb26aa98a1c3977077ad568e11ad" }, "costs": { "agent_known_micro_usd": 1280000, "judge_known_micro_usd": 0, "known_subtotal_micro_usd": 1280000, "unknown_component_count": 0, "judge_not_dispatched_trials": 16, "complete_selected_estimate_micro_usd": 1280000, "measurement_kind": "synthetic", "scope": "new agent/judge dispatch estimates in this packet; excludes prior attempts, tool/hosting/storage/human costs and invoice reconciliation", "deployment_authorized": false }, "candidate_contract_passes": 8, "candidate_completed_tasks": 6, "unqualified_message_trials": 16 } ], "limitations": "Four scripted cases share one customer; two permutations are not independent samples. Structural contract only, not qualified prose. Costs/latency profile invented; actual mock elapsed time retained separately. No live models, human review, execution attestation, or deployment qualification." }