{
  "schemaVersion": "0.1",
  "frameworkVersion": "0.2.0",
  "id": "15dfe11b-2e6d-4f47-b676-601f2ffa37d0",
  "manifest": {
    "schemaVersion": "0.1",
    "id": "production-api-handoff-2026-09-08",
    "name": "Finding the right API workflow — frozen production responses",
    "hypothesis": "A thin navigation index helps an agent reconcile public study evidence and prepare a correct API/CLI handoff from identical frozen production response files. Only navigation/progressive context is changed. Fixed 4-pair pilot; findings apply only to this task, source capture, requested runtime and budget, not live network or general agent performance.",
    "task": {
      "id": "reconcile-evidence-and-prepare-account-handoff",
      "version": "1",
      "prompt": "Prepare a production-platform handoff using the frozen public Evals AX HTTP responses and full documentation in this workspace. This is a read-only source/response task, not live HTTP execution. We want to decide what the two public study records demonstrate, determine whether those exact public JSON records can serve as full uploaded experiment results, and plan the correct authenticated account workflow for a separate genuine local result. Do not execute mutations, authenticate, use credentials, access the network, change files, read outside this workspace or seek evaluator/results/session files. Return exactly one JSON object following ANSWER_FORMAT.json. Reconcile both study inventories, compute the requested descriptive duration/token quantities, and preserve missing cost and inference limits. Identify every required top-level ExperimentResultInput field absent from each public record; do not invent missing evidence. Provide the ordered six API operations using complete /api/v1 paths and the top-level required request body keys from the OpenAPI contract (resolve references). Assume the host will authenticate separately, the account has authority to agree only after reviewing current terms, a new project is desired, a genuine manifest is available, and a full local result ./results/result.json becomes available later. Use PROJECT_ID as the CLI project placeholder and {id} in API paths. Supply verbatim quotations from production/full.md for terms authority/review, result immutability, and the scope of the inconclusive evidence. All source information remains available; choose your reading strategy."
    },
    "surfaces": {
      "baseline": {
        "path": "baseline",
        "revision": "159e1eb-production-responses-generic-entrypoint-e8c6323ce9b69d831f185fb304ff128a087303e9e0d425bfcd0ed7a995bc4924"
      },
      "candidate": {
        "path": "candidate",
        "revision": "159e1eb-production-responses-navigation-index-3fb7fe34c149d81d904b7e505a1ac7697fa42ccaa6c3e9da696e01a188ba4db2"
      }
    },
    "agent": {
      "kind": "codex",
      "model": "gpt-6-astra",
      "reasoning": "low",
      "executable": "/Applications/ChatGPT.app/Contents/Resources/codex",
      "configuration": "isolated"
    },
    "verifier": {
      "command": [
        "python3",
        "verify.py"
      ],
      "version": "1",
      "files": [
        "verify.py",
        "expected.json",
        "baseline/production/full.md",
        "baseline/production/openapi.json",
        "baseline/production/auth-study.json",
        "baseline/production/dispatch-study.json",
        "baseline/production/http-observations.json"
      ],
      "controls": [
        {
          "id": "complete-handoff",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
          "expectedPass": true
        },
        {
          "id": "unknown-cost-is-not-zero",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":0,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
          "expectedPass": false
        },
        {
          "id": "projection-is-not-full-result",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":true,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
          "expectedPass": false
        },
        {
          "id": "terms-input-needs-accepted",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
          "expectedPass": false
        },
        {
          "id": "wrong-study-arithmetic",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476973,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
          "expectedPass": false
        },
        {
          "id": "unsubstantiated-quote",
          "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"A source index always improves every agent task.\"}]}",
          "expectedPass": false
        }
      ]
    },
    "design": {
      "repetitions": 4,
      "seed": 2026090801,
      "minimumEffect": 0
    },
    "limits": {
      "trialTimeoutMs": 120000,
      "verifierTimeoutMs": 5000,
      "totalTimeoutMs": 1080000,
      "maxOutputBytes": 2000000,
      "maxSurfaceBytes": 1000000,
      "maxSurfaceFiles": 50
    }
  },
  "manifestHash": "c746dd4e9df8f01a446c584bcd78cf61aed58137ff6585139efd37d902534f29",
  "startedAt": "2026-09-08T08:31:21.533Z",
  "finishedAt": "2026-09-08T08:45:01.879Z",
  "completion": "complete",
  "provenance": {
    "runtime": "v24.6.0",
    "platform": "darwin/arm64",
    "agentVersion": "codex-cli 0.153.4",
    "surfaces": {
      "baseline": "cf38feb8b4166b7783524315ac2a3fd7cabb68f1420d1a330e3f1a066253a3d8",
      "candidate": "7ec1d111a0a071d9173526b452d068e1e5c2199b729875900437298411b9e494"
    },
    "verifier": [
      {
        "path": "verify.py",
        "sha256": "a528400d84d516370f0f999b9eb6f2f66a8793907ccc4603df92d32f673cbdfd"
      },
      {
        "path": "expected.json",
        "sha256": "079463aea0501865527bf30c9603310b0b0e29c49f39e4407b1b74679af61cd8"
      },
      {
        "path": "baseline/production/full.md",
        "sha256": "2381575b7512e2b8f37ee488ef2be7aeedeafaf9fe4a0c27bc61127443a7f6e7"
      },
      {
        "path": "baseline/production/openapi.json",
        "sha256": "6f0bb187bc2bb8e3e0bedbd74bfbe487b62edb3212ca456f08158b084d7f288f"
      },
      {
        "path": "baseline/production/auth-study.json",
        "sha256": "cb8e1cb5db90f9cad1c4d62b568389b5b619e65bc93b260e1616ae94371fa0ae"
      },
      {
        "path": "baseline/production/dispatch-study.json",
        "sha256": "2510d98e57db4b9d833e5adac79bb716757d5ea4ce9b5f6fb0bd1438bbf95d67"
      },
      {
        "path": "baseline/production/http-observations.json",
        "sha256": "b669b3c3e5078b73ceffa6f5e0e68d35227d8a99e71c9e9562e19ab3687bd36e"
      }
    ],
    "verifierControls": [
      {
        "id": "complete-handoff",
        "expectedPass": true,
        "observedPass": true,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 41.44424999999998,
        "trace": {
          "file": "control-complete-handoff.trace.json",
          "sha256": "8f89e390a3390b9b22db35eae6d272b600a281a05a8e0936118f45044c5f3202",
          "bytes": 162
        }
      },
      {
        "id": "unknown-cost-is-not-zero",
        "expectedPass": false,
        "observedPass": false,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 43.69012500000002,
        "trace": {
          "file": "control-unknown-cost-is-not-zero.trace.json",
          "sha256": "be75c7544a5cd0955158707a70f95e208717ed76ab2045ab8221191a8f3d4185",
          "bytes": 179
        }
      },
      {
        "id": "projection-is-not-full-result",
        "expectedPass": false,
        "observedPass": false,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 42.197541,
        "trace": {
          "file": "control-projection-is-not-full-result.trace.json",
          "sha256": "b53ed949da0b3c22b077bd681ca256fbb0888527609342b1d4e5d0bae5c14874",
          "bytes": 190
        }
      },
      {
        "id": "terms-input-needs-accepted",
        "expectedPass": false,
        "observedPass": false,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 41.86983299999997,
        "trace": {
          "file": "control-terms-input-needs-accepted.trace.json",
          "sha256": "2c38ff6117f8e39e5c63ab23613a6e03e86e3638b738970c64439fff4b87ae26",
          "bytes": 155
        }
      },
      {
        "id": "wrong-study-arithmetic",
        "expectedPass": false,
        "observedPass": false,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 42.63908299999997,
        "trace": {
          "file": "control-wrong-study-arithmetic.trace.json",
          "sha256": "4e75cbea789eab700a8145622c548313d222f75b4ff6f7c79a7028a673060362",
          "bytes": 160
        }
      },
      {
        "id": "unsubstantiated-quote",
        "expectedPass": false,
        "observedPass": false,
        "outcome": "matched",
        "reason": "Verifier matched this declared control.",
        "durationMs": 41.38654100000002,
        "trace": {
          "file": "control-unsubstantiated-quote.trace.json",
          "sha256": "c8806e0a233b5ddf6e57ac0a43a57acaa3e82f1afcdc52bec196c7dba684595e",
          "bytes": 214
        }
      }
    ],
    "limitations": [
      "Evidence was produced on a client host. A cloud upload does not independently attest execution or truth of results.",
      "Copied snapshots omit .git, node_modules and .evx directories. Symlinks and special files are rejected.",
      "Common credential filenames cause snapshot preflight failure; this is not a comprehensive secret detector. Inspect fixture contents before sending them to an agent.",
      "Fresh workspaces do not reset host agent settings, caches, network services or physical environments. Trials must avoid persistent side effects.",
      "Codex uses read-only sandboxing. Claude is limited to Read, Glob and Grep. Custom commands and verifiers execute with the caller's host permissions.",
      "Time and output caps are enforced locally. Codex does not expose a per-run monetary cap; its monetary cost is unknown. Claude's configured monetary cap is provider-enforced.",
      "The verifier is a separate local process. Its declared files are hashed and checked for changes; this is not protection from a hostile host process."
    ]
  },
  "trials": [
    {
      "id": "1-candidate",
      "pair": 0,
      "variant": "candidate",
      "order": 0,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:31:21.841Z",
      "durationMs": 106381.706416,
      "agentDurationMs": 106321.029708,
      "usage": {
        "inputTokens": 215139,
        "cachedInputTokens": 146048,
        "outputTokens": 2281,
        "reasoningTokens": 36,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 11,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "1-candidate.trace.json",
        "sha256": "44d8fb2b17b7c221625b96c4f925211cd6707c17ae68121ddc89ac0f1d86391a",
        "bytes": 505371
      },
      "outputHash": "358e7e9427fee2d92274e2761a900eb20c28fb034ec7f94a691e3a4eebcdc4aa",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    },
    {
      "id": "1-baseline",
      "pair": 0,
      "variant": "baseline",
      "order": 1,
      "status": "failed",
      "reason": "sources.scope: quotation absent from frozen source; sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:33:08.230Z",
      "durationMs": 115883.715292,
      "agentDurationMs": 115822.92279100002,
      "usage": {
        "inputTokens": 243214,
        "cachedInputTokens": 207744,
        "outputTokens": 2345,
        "reasoningTokens": 71,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 11,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "1-baseline.trace.json",
        "sha256": "f0268bf9db910b20373196d71dc3ca323642b4ab65762695792e49b7b3851516",
        "bytes": 454310
      },
      "outputHash": "81bb349eda6a5fb47c3f358c11687c6ba77be824f4213e97aa3b5e6f6c99191b",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation absent from frozen source; sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 2
        }
      }
    },
    {
      "id": "2-baseline",
      "pair": 1,
      "variant": "baseline",
      "order": 2,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:35:04.120Z",
      "durationMs": 106190.53033300003,
      "agentDurationMs": 106127.61379199999,
      "usage": {
        "inputTokens": 193084,
        "cachedInputTokens": 161408,
        "outputTokens": 2268,
        "reasoningTokens": 28,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 9,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "2-baseline.trace.json",
        "sha256": "9e6deb520d147951124278f09dc0f5135907342f193ebcfbd326584c249f52da",
        "bytes": 286331
      },
      "outputHash": "50ee465afb8b0f0ae2eb09b07db06e8d5b2ed64a262f0a41dfbef9134c8463dd",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    },
    {
      "id": "2-candidate",
      "pair": 1,
      "variant": "candidate",
      "order": 3,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:36:50.316Z",
      "durationMs": 95671.18350000004,
      "agentDurationMs": 95607.83937500004,
      "usage": {
        "inputTokens": 194568,
        "cachedInputTokens": 165248,
        "outputTokens": 2067,
        "reasoningTokens": 47,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 9,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "2-candidate.trace.json",
        "sha256": "2628ae85f54117b171a1308351cb2421de8cf62e553a3ae7ad0c8d87e6d08e4e",
        "bytes": 444488
      },
      "outputHash": "50ee465afb8b0f0ae2eb09b07db06e8d5b2ed64a262f0a41dfbef9134c8463dd",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    },
    {
      "id": "3-candidate",
      "pair": 2,
      "variant": "candidate",
      "order": 4,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:38:25.994Z",
      "durationMs": 89160.14566600003,
      "agentDurationMs": 89096.70545800001,
      "usage": {
        "inputTokens": 217246,
        "cachedInputTokens": 179712,
        "outputTokens": 2003,
        "reasoningTokens": 31,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 8,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "3-candidate.trace.json",
        "sha256": "16fff00fb0d9adf147862d4b8056912657da6b16b57308ce4fdb11254fb02a04",
        "bytes": 456900
      },
      "outputHash": "50ee465afb8b0f0ae2eb09b07db06e8d5b2ed64a262f0a41dfbef9134c8463dd",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    },
    {
      "id": "3-baseline",
      "pair": 2,
      "variant": "baseline",
      "order": 5,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:39:55.160Z",
      "durationMs": 110176.84908400004,
      "agentDurationMs": 110116.500458,
      "usage": {
        "inputTokens": 237281,
        "cachedInputTokens": 207488,
        "outputTokens": 2339,
        "reasoningTokens": 74,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 11,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "3-baseline.trace.json",
        "sha256": "7f96f87958ff413d1603cede053435d19254cbd2c2f1cc212b7ec278cd2aba22",
        "bytes": 456365
      },
      "outputHash": "aa91bd6b13a17a20f50b5880e844f99521db550db2f4a1eef152a1372600b4b0",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    },
    {
      "id": "4-baseline",
      "pair": 3,
      "variant": "baseline",
      "order": 6,
      "status": "failed",
      "reason": "answer.policy.validateCommand: incorrect CLI workflow; sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:41:45.344Z",
      "durationMs": 95642.06641700002,
      "agentDurationMs": 95577.63791699999,
      "usage": {
        "inputTokens": 194944,
        "cachedInputTokens": 165760,
        "outputTokens": 2140,
        "reasoningTokens": 17,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 9,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "4-baseline.trace.json",
        "sha256": "81ae92d05d2cc926b5be8dd83d1223ce8725ce42a4dbdfdc395179223c60d282",
        "bytes": 445505
      },
      "outputHash": "ba899f215df29105d8353fa6e5036b35b7ced7d4d546fb8865c7c7ed4a5281fd",
      "verification": {
        "passed": false,
        "reason": "answer.policy.validateCommand: incorrect CLI workflow; sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 2
        }
      }
    },
    {
      "id": "4-candidate",
      "pair": 3,
      "variant": "candidate",
      "order": 7,
      "status": "failed",
      "reason": "sources.scope: quotation does not cover the required boundary",
      "startedAt": "2026-09-08T08:43:20.994Z",
      "durationMs": 100881.51404099993,
      "agentDurationMs": 100834.882125,
      "usage": {
        "inputTokens": 197566,
        "cachedInputTokens": 171008,
        "outputTokens": 2108,
        "reasoningTokens": 21,
        "costUsd": null,
        "source": "agent-reported"
      },
      "diagnostics": {
        "commandCalls": 8,
        "nonzeroCommandExits": 0,
        "events": []
      },
      "trace": {
        "file": "4-candidate.trace.json",
        "sha256": "ba7f0a5e94abc4a3ab6e84bf75e3ca06707e6d124416ab252bc36e1948a78b49",
        "bytes": 253683
      },
      "outputHash": "50ee465afb8b0f0ae2eb09b07db06e8d5b2ed64a262f0a41dfbef9134c8463dd",
      "verification": {
        "passed": false,
        "reason": "sources.scope: quotation does not cover the required boundary",
        "metrics": {
          "failedChecks": 1
        }
      }
    }
  ]
}