{"kind":"public-study-projection","schemaVersion":"1","title":"Finding the right API workflow — frozen production responses","reportId":"15dfe11b-2e6d-4f47-b676-601f2ffa37d0","manifestHash":"c746dd4e9df8f01a446c584bcd78cf61aed58137ff6585139efd37d902534f29","originalResultSha256":"3176505ee2526d46d9425cb5b0b548a09313db9812c29938efcd7f32f67db7ba","canonicalReportFingerprint":"e72f9aeb7bbc7c823956056e265938e9677c1bb2e20f1d2cd22fd7199e73561d","fingerprintMethod":"sha256 of schema-validated JSON with recursively sorted object keys; array order retained","sourceCommit":"8d86dda1be42db57df4c8b82cecd1749ad04bc42","observedProductionCommit":"159e1eb55fd87d99d399de63e12c65292f621b78","planRecordedAt":"2026-09-08T08:29:41.424Z","startedAt":"2026-09-08T08:31:21.533Z","finishedAt":"2026-09-08T08:45:01.879Z","evidenceOrigin":"client-reported","agent":{"kind":"codex","requestedModel":"gpt-6-astra","reasoning":"low","configuration":"isolated","runtimeVersion":"codex-cli 0.153.4"},"scope":"Real agent execution on frozen production response files, prepared for production-platform upload. No per-trial HTTP, authenticated account operations, browser or assistive-technology coverage.","design":{"pairs":4,"plannedTrials":8,"trialTimeoutMs":120000,"conditions":{"baseline":"Generic entry point","candidate":"Navigation index"},"change":"Only candidate AGENTS.md adds a navigation index; ANSWER_FORMAT.json and every production response byte are identical."},"original":{"verifierVersion":"1","interpretation":"Known verifier defect; these verdicts do not establish agent success rates.","baselinePassed":0,"candidatePassed":0,"plannedPerCondition":4,"unchanged":true},"sensitivity":{"kind":"post-hoc-verifier-sensitivity","verifierVersion":"2","planRecordedAt":"2026-09-08T08:38:28.948294+00:00","trialsCompletedBeforeRevision":4,"assessedTrials":8,"baselineAccepted":2,"candidateAccepted":4,"repairedFalseNegatives":6,"defect":"Version 1 required the word inconclusive in a scope quotation. It rejected source-grounded statements that explicitly denied generalization using different wording. A failed original verdict is therefore not automatically an agent failure.","repair":"Version 2 accepts three equivalent statements found in the frozen source while preserving all other checks. Original verdicts and source files are unchanged. Every retained answer was assessed under both versions, including failures.","inference":"This is a post-hoc verifier diagnosis and sensitivity analysis, not a preregistered success comparison. Neither the flawed original predicate nor the repaired counts establish that the navigation index improves agent performance.","sampleLimit":"Four pairs are too few for a general claim, and the verifier changed after outcomes were observed.","retainedDefects":[{"trialId":"1-baseline","description":"Combined noncontiguous passages into a purported verbatim quotation."},{"trialId":"4-baseline","description":"Supplied an unsupported CLI validation flag."}]},"trials":[{"id":"1-candidate","variant":"candidate","order":0,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":106321.029708,"inputTokens":215139,"outputTokens":2281,"costUsd":null},{"id":"1-baseline","variant":"baseline","order":1,"originalStatus":"failed","postHocStatus":"failed","classification":"retained-verifier-failure","agentDurationMs":115822.92279100002,"inputTokens":243214,"outputTokens":2345,"costUsd":null},{"id":"2-baseline","variant":"baseline","order":2,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":106127.613792,"inputTokens":193084,"outputTokens":2268,"costUsd":null},{"id":"2-candidate","variant":"candidate","order":3,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":95607.83937500004,"inputTokens":194568,"outputTokens":2067,"costUsd":null},{"id":"3-candidate","variant":"candidate","order":4,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":89096.70545800001,"inputTokens":217246,"outputTokens":2003,"costUsd":null},{"id":"3-baseline","variant":"baseline","order":5,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":110116.500458,"inputTokens":237281,"outputTokens":2339,"costUsd":null},{"id":"4-baseline","variant":"baseline","order":6,"originalStatus":"failed","postHocStatus":"failed","classification":"retained-verifier-failure","agentDurationMs":95577.637917,"inputTokens":194944,"outputTokens":2140,"costUsd":null},{"id":"4-candidate","variant":"candidate","order":7,"originalStatus":"failed","postHocStatus":"passed","classification":"verifier-false-negative","agentDurationMs":100834.882125,"inputTokens":197566,"outputTokens":2108,"costUsd":null}],"measurements":{"interpretation":"Duration and usage are descriptive only; no efficiency inference was registered.","baseline":{"meanAgentDurationMs":106911.1687395,"inputTokens":868523,"inputTokenReportedTrials":4,"outputTokens":9092,"outputTokenReportedTrials":4,"costUsd":null,"costReportedTrials":0,"costComplete":false},"candidate":{"meanAgentDurationMs":97965.11416650002,"inputTokens":824519,"inputTokenReportedTrials":4,"outputTokens":8459,"outputTokenReportedTrials":4,"costUsd":null,"costReportedTrials":0,"costComplete":false}},"isolationLimit":"Fresh local workspaces and retained provider traces are not external filesystem or complete-model-context attestation. Cloud storage does not add such attestation.","projectionLimit":"This is a public review projection, not a full ExperimentResult or an uploadable replacement for the original report."}