{
  "schemaVersion": "0.1",
  "id": "production-api-handoff-2026-09-08",
  "name": "Finding the right API workflow — frozen production responses",
  "hypothesis": "A thin navigation index helps an agent reconcile public study evidence and prepare a correct API/CLI handoff from identical frozen production response files. Only navigation/progressive context is changed. Fixed 4-pair pilot; findings apply only to this task, source capture, requested runtime and budget, not live network or general agent performance.",
  "task": {
    "id": "reconcile-evidence-and-prepare-account-handoff",
    "version": "1",
    "prompt": "Prepare a production-platform handoff using the frozen public Evals AX HTTP responses and full documentation in this workspace. This is a read-only source/response task, not live HTTP execution. We want to decide what the two public study records demonstrate, determine whether those exact public JSON records can serve as full uploaded experiment results, and plan the correct authenticated account workflow for a separate genuine local result. Do not execute mutations, authenticate, use credentials, access the network, change files, read outside this workspace or seek evaluator/results/session files. Return exactly one JSON object following ANSWER_FORMAT.json. Reconcile both study inventories, compute the requested descriptive duration/token quantities, and preserve missing cost and inference limits. Identify every required top-level ExperimentResultInput field absent from each public record; do not invent missing evidence. Provide the ordered six API operations using complete /api/v1 paths and the top-level required request body keys from the OpenAPI contract (resolve references). Assume the host will authenticate separately, the account has authority to agree only after reviewing current terms, a new project is desired, a genuine manifest is available, and a full local result ./results/result.json becomes available later. Use PROJECT_ID as the CLI project placeholder and {id} in API paths. Supply verbatim quotations from production/full.md for terms authority/review, result immutability, and the scope of the inconclusive evidence. All source information remains available; choose your reading strategy."
  },
  "surfaces": {
    "baseline": {
      "path": "baseline",
      "revision": "159e1eb-production-responses-generic-entrypoint-e8c6323ce9b69d831f185fb304ff128a087303e9e0d425bfcd0ed7a995bc4924"
    },
    "candidate": {
      "path": "candidate",
      "revision": "159e1eb-production-responses-navigation-index-3fb7fe34c149d81d904b7e505a1ac7697fa42ccaa6c3e9da696e01a188ba4db2"
    }
  },
  "agent": {
    "kind": "codex",
    "model": "gpt-6-astra",
    "reasoning": "low",
    "executable": "/Applications/ChatGPT.app/Contents/Resources/codex",
    "configuration": "isolated"
  },
  "verifier": {
    "command": [
      "python3",
      "verify.py"
    ],
    "version": "1",
    "files": [
      "verify.py",
      "expected.json",
      "baseline/production/full.md",
      "baseline/production/openapi.json",
      "baseline/production/auth-study.json",
      "baseline/production/dispatch-study.json",
      "baseline/production/http-observations.json"
    ],
    "controls": [
      {
        "id": "complete-handoff",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
        "expectedPass": true
      },
      {
        "id": "unknown-cost-is-not-zero",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":0,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
        "expectedPass": false
      },
      {
        "id": "projection-is-not-full-result",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":true,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
        "expectedPass": false
      },
      {
        "id": "terms-input-needs-accepted",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
        "expectedPass": false
      },
      {
        "id": "wrong-study-arithmetic",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476973,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"This is inconclusive, not evidence that the two interfaces are equivalent.\"}]}",
        "expectedPass": false
      },
      {
        "id": "unsubstantiated-quote",
        "output": "{\"studies\":[{\"id\":\"evals-auth-context-index-2026-09-07\",\"plannedTrials\":24,\"passedTrials\":24,\"conclusion\":\"inconclusive\",\"effectInterval\":[-0.7841002756996853,0.7841002756996853],\"candidateMinusBaselineMeanMs\":7140.713881749984,\"totalInputTokens\":1476972,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]},{\"id\":\"39eebf9f-b347-4c35-a6b3-81929c0d8927\",\"plannedTrials\":8,\"passedTrials\":8,\"conclusion\":\"inconclusive\",\"effectInterval\":[-1,1],\"candidateMinusBaselineMeanMs\":11964.364812499996,\"totalInputTokens\":509509,\"costUsd\":null,\"supportsGeneralRollout\":false,\"uploadableAsResult\":false,\"missingRequiredResultFields\":[\"completion\",\"manifest\",\"provenance\",\"schemaVersion\",\"trials\"]}],\"combined\":{\"plannedTrials\":32,\"passedTrials\":32,\"supportsPooledCausalConclusion\":false},\"access\":{\"accountStatus\":401,\"accountErrorCode\":\"sign_in_required\",\"loginCommand\":\"evx auth login\"},\"workflow\":[{\"step\":\"readTerms\",\"method\":\"GET\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[]},{\"step\":\"acceptReviewedTerms\",\"method\":\"PUT\",\"path\":\"/api/v1/account/terms\",\"requiredBodyKeys\":[\"accepted\",\"version\"]},{\"step\":\"createProject\",\"method\":\"POST\",\"path\":\"/api/v1/projects\",\"requiredBodyKeys\":[\"name\"]},{\"step\":\"createExperiment\",\"method\":\"POST\",\"path\":\"/api/v1/experiments\",\"requiredBodyKeys\":[\"manifest\",\"projectId\",\"title\"]},{\"step\":\"attachResult\",\"method\":\"PUT\",\"path\":\"/api/v1/experiments/{id}\",\"requiredBodyKeys\":[\"report\"]},{\"step\":\"exportResult\",\"method\":\"GET\",\"path\":\"/api/v1/experiments/{id}/result\",\"requiredBodyKeys\":[]}],\"policy\":{\"termsAcceptedValue\":true,\"termsNeedExplicitReview\":true,\"staleTermsRetryAutomatically\":false,\"replaceExistingResultAllowed\":false,\"identicalResultRetryAllowed\":true,\"uploadMaxBytes\":2000000,\"validateCommand\":\"evx experiment validate ./results/result.json\",\"uploadCommand\":\"evx experiment upload ./results/result.json --project PROJECT_ID\"},\"sources\":[{\"claim\":\"terms\",\"file\":\"production/full.md\",\"quote\":\"Do not automatically accept on behalf of someone who has not authorized you.\"},{\"claim\":\"immutable\",\"file\":\"production/full.md\",\"quote\":\"An uploaded result is immutable.\"},{\"claim\":\"scope\",\"file\":\"production/full.md\",\"quote\":\"A source index always improves every agent task.\"}]}",
        "expectedPass": false
      }
    ]
  },
  "design": {
    "repetitions": 4,
    "seed": 2026090801,
    "minimumEffect": 0
  },
  "limits": {
    "trialTimeoutMs": 120000,
    "verifierTimeoutMs": 5000,
    "totalTimeoutMs": 1080000,
    "maxOutputBytes": 2000000,
    "maxSurfaceBytes": 1000000,
    "maxSurfaceFiles": 50
  }
}