{ "schemaVersion": "0.1", "frameworkVersion": "0.1.0", "id": "39eebf9f-b347-4c35-a6b3-81929c0d8927", "manifest": { "schemaVersion": "0.1", "id": "dispatch-reconciliation", "name": "Reconcile a warehouse dispatch from inconsistent exports", "hypothesis": "A concise canonical-source and join-key index may improve completion of a multi-file reconciliation task. Both conditions contain identical operational facts and policy. This experiment tests this particular navigation change, not a universal rule about context length.", "task": { "id": "prepare-noon-dispatch", "version": "1", "prompt": "Prepare the noon warehouse dispatch handoff from this workspace. Reconcile all order lines, payment/refund events, stock movements and customer hold history under the current policy. Produce a complete order ledger, every policy-defined data exception, and the safe whole-order dispatch plan. This is an offline review; do not call external services or change files.\nReturn exactly one JSON object with three keys:\n- \"ledger.csv\": a CSV string, header order_id,total_minor,net_paid_minor,decision, one row per order sorted by order_id; decisions are invalid_sku, cancelled, held, unpaid, backorder, or ship.\n- \"exceptions.json\": an array of objects with exactly kind, source, recordId for each policy-defined exception. Order does not matter.\n- \"dispatch.json\": an object with exactly cutoff (ISO timestamp), shipments (array in allocation order, each {orderId,items:[{sku,quantity}]} using canonical SKUs), and remainingStock (object containing every canonical SKU and its final integer quantity). Sort shipment items by sku.\nNo markdown fences or additional commentary. All three outputs are required, including orders that must not ship." }, "surfaces": { "baseline": { "path": "baseline", "revision": "warehouse-v1-generic-entrypoint" }, "candidate": { "path": "candidate", "revision": "warehouse-v1-source-and-join-index" } }, "agent": { "kind": "codex", "model": "gpt-6-astra", "reasoning": "low", "executable": "/Applications/ChatGPT.app/Contents/Resources/codex", "configuration": "isolated" }, "verifier": { "command": [ "node", "verify.mjs" ], "version": "1", "files": [ "verify.mjs", "validation.mjs", "expected.json" ], "controls": [ { "id": "all-reconciled", "output": "{\"ledger.csv\":\"order_id,total_minor,net_paid_minor,decision\\nO101,3000,3000,ship\\nO102,4000,4000,held\\nO103,2000,2000,ship\\nO104,5000,0,unpaid\\nO105,6000,6000,ship\\nO106,6000,6000,cancelled\\nO107,4000,4000,backorder\\nO108,1500,1500,ship\\nO109,5000,5000,ship\\nO110,1000,1000,invalid_sku\\n\",\"exceptions.json\":[{\"kind\":\"duplicate_event\",\"source\":\"exports/payments.csv\",\"recordId\":\"P10\"},{\"kind\":\"unknown_order\",\"source\":\"exports/payments.csv\",\"recordId\":\"P12\"},{\"kind\":\"currency_mismatch\",\"source\":\"exports/payments.csv\",\"recordId\":\"P13\"},{\"kind\":\"duplicate_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S01\"},{\"kind\":\"conflicting_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S09\"},{\"kind\":\"unknown_sku\",\"source\":\"exports/order-lines.csv\",\"recordId\":\"L12\"}],\"dispatch.json\":{\"cutoff\":\"2026-06-03T12:00:00Z\",\"shipments\":[{\"orderId\":\"O101\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":2},{\"sku\":\"HUB\",\"quantity\":1}]},{\"orderId\":\"O103\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":4}]},{\"orderId\":\"O105\",\"items\":[{\"sku\":\"HUB\",\"quantity\":3}]},{\"orderId\":\"O108\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":3}]},{\"orderId\":\"O109\",\"items\":[{\"sku\":\"DOCK\",\"quantity\":1}]}],\"remainingStock\":{\"CABLE\":0,\"DOCK\":1,\"HUB\":1}}}", "expectedPass": true }, { "id": "unpaid-shipment", "output": "{\"ledger.csv\":\"order_id,total_minor,net_paid_minor,decision\\nO101,3000,3000,ship\\nO102,4000,4000,held\\nO103,2000,2000,ship\\nO104,5000,0,unpaid\\nO105,6000,6000,ship\\nO106,6000,6000,cancelled\\nO107,4000,4000,backorder\\nO108,1500,1500,ship\\nO109,5000,5000,ship\\nO110,1000,1000,invalid_sku\\n\",\"exceptions.json\":[{\"kind\":\"duplicate_event\",\"source\":\"exports/payments.csv\",\"recordId\":\"P10\"},{\"kind\":\"unknown_order\",\"source\":\"exports/payments.csv\",\"recordId\":\"P12\"},{\"kind\":\"currency_mismatch\",\"source\":\"exports/payments.csv\",\"recordId\":\"P13\"},{\"kind\":\"duplicate_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S01\"},{\"kind\":\"conflicting_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S09\"},{\"kind\":\"unknown_sku\",\"source\":\"exports/order-lines.csv\",\"recordId\":\"L12\"}],\"dispatch.json\":{\"cutoff\":\"2026-06-03T12:00:00Z\",\"shipments\":[{\"orderId\":\"O101\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":2},{\"sku\":\"HUB\",\"quantity\":1}]},{\"orderId\":\"O103\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":4}]},{\"orderId\":\"O105\",\"items\":[{\"sku\":\"HUB\",\"quantity\":3}]},{\"orderId\":\"O108\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":3}]},{\"orderId\":\"O109\",\"items\":[{\"sku\":\"DOCK\",\"quantity\":1}]},{\"orderId\":\"O104\",\"items\":[{\"sku\":\"DOCK\",\"quantity\":1}]}],\"remainingStock\":{\"CABLE\":0,\"DOCK\":1,\"HUB\":1}}}", "expectedPass": false }, { "id": "missing-exception", "output": "{\"ledger.csv\":\"order_id,total_minor,net_paid_minor,decision\\nO101,3000,3000,ship\\nO102,4000,4000,held\\nO103,2000,2000,ship\\nO104,5000,0,unpaid\\nO105,6000,6000,ship\\nO106,6000,6000,cancelled\\nO107,4000,4000,backorder\\nO108,1500,1500,ship\\nO109,5000,5000,ship\\nO110,1000,1000,invalid_sku\\n\",\"exceptions.json\":[{\"kind\":\"duplicate_event\",\"source\":\"exports/payments.csv\",\"recordId\":\"P10\"},{\"kind\":\"unknown_order\",\"source\":\"exports/payments.csv\",\"recordId\":\"P12\"},{\"kind\":\"currency_mismatch\",\"source\":\"exports/payments.csv\",\"recordId\":\"P13\"},{\"kind\":\"duplicate_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S01\"},{\"kind\":\"conflicting_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S09\"}],\"dispatch.json\":{\"cutoff\":\"2026-06-03T12:00:00Z\",\"shipments\":[{\"orderId\":\"O101\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":2},{\"sku\":\"HUB\",\"quantity\":1}]},{\"orderId\":\"O103\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":4}]},{\"orderId\":\"O105\",\"items\":[{\"sku\":\"HUB\",\"quantity\":3}]},{\"orderId\":\"O108\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":3}]},{\"orderId\":\"O109\",\"items\":[{\"sku\":\"DOCK\",\"quantity\":1}]}],\"remainingStock\":{\"CABLE\":0,\"DOCK\":1,\"HUB\":1}}}", "expectedPass": false }, { "id": "double-counted-stock", "output": "{\"ledger.csv\":\"order_id,total_minor,net_paid_minor,decision\\nO101,3000,3000,ship\\nO102,4000,4000,held\\nO103,2000,2000,ship\\nO104,5000,0,unpaid\\nO105,6000,6000,ship\\nO106,6000,6000,cancelled\\nO107,4000,4000,backorder\\nO108,1500,1500,ship\\nO109,5000,5000,ship\\nO110,1000,1000,invalid_sku\\n\",\"exceptions.json\":[{\"kind\":\"duplicate_event\",\"source\":\"exports/payments.csv\",\"recordId\":\"P10\"},{\"kind\":\"unknown_order\",\"source\":\"exports/payments.csv\",\"recordId\":\"P12\"},{\"kind\":\"currency_mismatch\",\"source\":\"exports/payments.csv\",\"recordId\":\"P13\"},{\"kind\":\"duplicate_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S01\"},{\"kind\":\"conflicting_event\",\"source\":\"exports/stock.csv\",\"recordId\":\"S09\"},{\"kind\":\"unknown_sku\",\"source\":\"exports/order-lines.csv\",\"recordId\":\"L12\"}],\"dispatch.json\":{\"cutoff\":\"2026-06-03T12:00:00Z\",\"shipments\":[{\"orderId\":\"O101\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":2},{\"sku\":\"HUB\",\"quantity\":1}]},{\"orderId\":\"O103\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":4}]},{\"orderId\":\"O105\",\"items\":[{\"sku\":\"HUB\",\"quantity\":3}]},{\"orderId\":\"O108\",\"items\":[{\"sku\":\"CABLE\",\"quantity\":3}]},{\"orderId\":\"O109\",\"items\":[{\"sku\":\"DOCK\",\"quantity\":1}]}],\"remainingStock\":{\"CABLE\":12,\"DOCK\":1,\"HUB\":1}}}", "expectedPass": false } ] }, "design": { "repetitions": 4, "seed": 20260908, "minimumEffect": 0 }, "limits": { "trialTimeoutMs": 100000, "verifierTimeoutMs": 5000, "totalTimeoutMs": 900000, "maxOutputBytes": 1000000, "maxSurfaceBytes": 1000000, "maxSurfaceFiles": 100 } }, "manifestHash": "53542810ac5f8f127fc3518b2e64f83dff0d6c94bd4c8a70be1309af9e194823", "startedAt": "2026-09-07T23:40:35.887Z", "finishedAt": "2026-09-07T23:48:22.495Z", "completion": "complete", "provenance": { "runtime": "v24.6.0", "platform": "darwin/arm64", "agentVersion": "codex-cli 0.153.4", "surfaces": { "baseline": "bada926c00a967b655ae95a08d9d6cc87bc5c0a371327697d846dbe49ef6b8e1", "candidate": "6cd8b28020b80e97c040e7a8d4ac49416054e0b9fd4febede9efc6cf33ec2a67" }, "verifier": [ { "path": "verify.mjs", "sha256": "2de997c9296f184b94e8320d9c9402f6256ccacf818fa2e81b1d0cbe3f311779" }, { "path": "validation.mjs", "sha256": "19a27d16ac0a9951101aa83c63d06135f8d7c7ba1a83057d37ee56f56865234d" }, { "path": "expected.json", "sha256": "4768cefed0a580224985aa870bee2629300eefb6e080c6e5dcb643bbfb4076e4" } ], "verifierControls": [ { "id": "all-reconciled", "expectedPass": true, "observedPass": true, "outcome": "matched", "reason": "Verifier matched this declared control.", "durationMs": 45.02366699999999, "trace": { "file": "control-all-reconciled.trace.json", "sha256": "a4ba81dc12370e25825737288ae3d10980aca1a009cf9a0eafe425c8b317218f", "bytes": 201 } }, { "id": "unpaid-shipment", "expectedPass": false, "observedPass": false, "outcome": "matched", "reason": "Verifier matched this declared control.", "durationMs": 45.891250000000014, "trace": { "file": "control-unpaid-shipment.trace.json", "sha256": "29f2f1920b4e4d73d43baf36a67ed15b55dc273906b4869dbebb836f9ab68fea", "bytes": 186 } }, { "id": "missing-exception", "expectedPass": false, "observedPass": false, "outcome": "matched", "reason": "Verifier matched this declared control.", "durationMs": 45.096834, "trace": { "file": "control-missing-exception.trace.json", "sha256": "34fa2c2b216aecfdc8fbf26bec7e84935ae4e5624767dbf889a00fb5e23f114b", "bytes": 188 } }, { "id": "double-counted-stock", "expectedPass": false, "observedPass": false, "outcome": "matched", "reason": "Verifier matched this declared control.", "durationMs": 45.04083300000002, "trace": { "file": "control-double-counted-stock.trace.json", "sha256": "29f2f1920b4e4d73d43baf36a67ed15b55dc273906b4869dbebb836f9ab68fea", "bytes": 186 } } ], "limitations": [ "Evidence was produced on a client host. A cloud upload does not independently attest execution or truth of results.", "Copied snapshots omit .git, node_modules and .evx directories. Symlinks and special files are rejected.", "Common credential filenames cause snapshot preflight failure; this is not a comprehensive secret detector. Inspect fixture contents before sending them to an agent.", "Fresh workspaces do not reset host agent settings, caches, network services or physical environments. Trials must avoid persistent side effects.", "Codex uses read-only sandboxing. Claude is limited to Read, Glob and Grep. Custom commands and verifiers execute with the caller's host permissions.", "Time and output caps are enforced locally. Codex does not expose a per-run monetary cap; its monetary cost is unknown. Claude's configured monetary cap is provider-enforced.", "The verifier is a separate local process. Its declared files are hashed and checked for changes; this is not protection from a hostile host process." ] }, "trials": [ { "id": "1-baseline", "pair": 0, "variant": "baseline", "order": 0, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:40:36.113Z", "durationMs": 64971.443208000004, "agentDurationMs": 64913.417457999996, "usage": { "inputTokens": 65246, "cachedInputTokens": 52736, "outputTokens": 1479, "reasoningTokens": 56, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 2, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "1-baseline.trace.json", "sha256": "577ff245b38b88e7266231c7e79d99e96616c0726ee1452ee9be9b4976cfa6b6", "bytes": 14314 }, "outputHash": "105d748ab64388c156459d458e4c986100130db65fa317f12ab3e343d417777b", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "1-candidate", "pair": 0, "variant": "candidate", "order": 1, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:41:41.087Z", "durationMs": 93031.15583300001, "agentDurationMs": 92978.49804099998, "usage": { "inputTokens": 86369, "cachedInputTokens": 77568, "outputTokens": 2281, "reasoningTokens": 40, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 9, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "1-candidate.trace.json", "sha256": "4be4b75520436e86f4fc91bc2d92232431d828699a0c26b8f5ebc9dcc86d56ee", "bytes": 26065 }, "outputHash": "f7314ff9030b2073f08b544c5c1dca97166d51389953bb69158178b84df6b4e9", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "2-candidate", "pair": 1, "variant": "candidate", "order": 2, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:43:14.122Z", "durationMs": 34164.22366700001, "agentDurationMs": 34113.52254200002, "usage": { "inputTokens": 47923, "cachedInputTokens": 41984, "outputTokens": 572, "reasoningTokens": 72, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 3, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "2-candidate.trace.json", "sha256": "aff492f91f1acfeccbbc3fb4c0cb3057e1d98b61512d1f5afe444e70a78216eb", "bytes": 13442 }, "outputHash": "7738ae1a9c4a0054620fb11918aa5d7e1eceb326f62b2264ab97a2af0563c691", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "2-baseline", "pair": 1, "variant": "baseline", "order": 3, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:43:48.288Z", "durationMs": 40139.617415999994, "agentDurationMs": 40080.761333, "usage": { "inputTokens": 47017, "cachedInputTokens": 37760, "outputTokens": 564, "reasoningTokens": 71, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 2, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "2-baseline.trace.json", "sha256": "a502b7b8c284ad69d8e383a55dffbebe09454c18a07634bdc1225c4fddb48e85", "bytes": 13469 }, "outputHash": "7738ae1a9c4a0054620fb11918aa5d7e1eceb326f62b2264ab97a2af0563c691", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "3-baseline", "pair": 2, "variant": "baseline", "order": 4, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:44:28.430Z", "durationMs": 35952.428916000004, "agentDurationMs": 35895.05799999999, "usage": { "inputTokens": 47200, "cachedInputTokens": 41216, "outputTokens": 642, "reasoningTokens": 96, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 2, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "3-baseline.trace.json", "sha256": "5575b1bc7bd5fcbbab09b50d8263820ac19866375461d8f4db541e40da20af4f", "bytes": 13914 }, "outputHash": "105d748ab64388c156459d458e4c986100130db65fa317f12ab3e343d417777b", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "3-candidate", "pair": 2, "variant": "candidate", "order": 5, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:45:04.385Z", "durationMs": 65011.38891700003, "agentDurationMs": 64951.567165999964, "usage": { "inputTokens": 65987, "cachedInputTokens": 55424, "outputTokens": 1464, "reasoningTokens": 84, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 3, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "3-candidate.trace.json", "sha256": "29dfea434d2fbe66c2bc28e8f19707d18cac6d90bcdf011eab2670964a7b785f", "bytes": 13765 }, "outputHash": "f7314ff9030b2073f08b544c5c1dca97166d51389953bb69158178b84df6b4e9", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "4-candidate", "pair": 3, "variant": "candidate", "order": 6, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:46:09.399Z", "durationMs": 64897.102249999996, "agentDurationMs": 64838.26329199999, "usage": { "inputTokens": 66140, "cachedInputTokens": 54912, "outputTokens": 1528, "reasoningTokens": 86, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 3, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "4-candidate.trace.json", "sha256": "4b16e450f6083bfef64dfb3ddc4104375540c1cad3caf742a5cc83a262e0fdff", "bytes": 13856 }, "outputHash": "3d1e5d3edcf3301e17f21fdc798aedc9c426d2a437e2fd64a73bbeccd089365e", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } }, { "id": "4-baseline", "pair": 3, "variant": "baseline", "order": 7, "status": "passed", "reason": "The independent reconciliation matches all three deliverables.", "startedAt": "2026-09-07T23:47:14.299Z", "durationMs": 68193.64466700004, "agentDurationMs": 68135.15499999997, "usage": { "inputTokens": 83627, "cachedInputTokens": 75904, "outputTokens": 1632, "reasoningTokens": 12, "costUsd": null, "source": "agent-reported" }, "diagnostics": { "commandCalls": 3, "nonzeroCommandExits": 0, "events": [] }, "trace": { "file": "4-baseline.trace.json", "sha256": "f16e8832153341cfb557dc67d9d5fc558b77e938d3e59e87607a89e3a19915a0", "bytes": 16361 }, "outputHash": "7738ae1a9c4a0054620fb11918aa5d7e1eceb326f62b2264ab97a2af0563c691", "verification": { "passed": true, "reason": "The independent reconciliation matches all three deliverables.", "metrics": { "ledgerCorrect": 1, "exceptionsCorrect": 1, "dispatchCorrect": 1 } } } ] }