{ "generated_at": "2026-09-11T19:13:18.771Z", "scope": "Vanguard order-service authoring pilot. Same source brief plus fixed author decisions and fixtures.", "source_generations": [ "5c2f60b0-ff9f-4ec6-9de7-365bb79bf0c2" ], "price_snapshot": { "date": "2026-09-11", "source": "https://developers.openai.com/api/docs/pricing", "rates": { "gpt-5.6-terra": { "input": 2, "cached": 0.2, "output": 12 }, "gpt-5.6-sol": { "input": 4, "cached": 0.4, "output": 20 }, "gpt-6-astra": { "input": 10, "cached": 1, "output": 50 } } }, "limitations": [ "A small pilot on one brief, not a general model leaderboard or statistically reliable tail-latency estimate.", "Active generation time excludes the controlled queue; includes tools, validation, final provider reply and cleanup. Hybrid time sums both measured stages, excluding the manual gap between benchmark launches.", "Claude Code and OpenAI Responses have different provider harnesses; separate Sol, Terra and Astra-low Agents rows measure the managed harness. Medium effort labels are not equal compute budgets across providers.", "OpenAI Agents usage may be null. Missing cost is never treated as free. Responses costs are list-price ranges, Claude costs are CLI list-price estimates; local infrastructure and evaluation costs are excluded.", "21 independent tests: 19 allow/deny outcomes plus two fixture-SKU disclosure checks added after a blind review found an untested wrong-postcode/wrong-item combination. The same tests run against every unchanged artifact in a sandbox with mock integration mode and no integration grants. They do not fully assert response content, real merchant actions or live voice calls.", "Saved sample specs count tools whose source, parameters and input schema still match their prepared specification and whose supplied test suite passes. Handwritten or altered tools can pass the independent checks while failing this separate integrity check.", "Prompt quality is one blinded Sol/high model review on an eight-criterion rubric, not a human quality rating.", "Review caveats preserve the original model score alongside any contradictory execution evidence. A reviewer claim is not automatically a verified defect.", "Provider aliases and account cache/load may change. Draft snapshots, prompts, raw usage and per-check results are retained locally." ], "rows": [ { "candidate": "astra", "repeat": "1", "authoring_model": "gpt-6-astra", "effort": "medium", "transport": "responses", "generation_id": "81f60b5c-07c2-4fd8-abda-18e9bc188c79", "target_id": "4359", "status": "completed", "excluded_reason": null, "source_file": "astra-1.json", "active_seconds": 486.55912566185, "review_stage_seconds": null, "queue_seconds": 489.48938965797424, "cost": { "min": 3.143204, "max": 3.3668240000000003, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 29, "issues": [] }, "tokens": { "input_total": 678072, "input_uncached": 89448, "input_cached": 588624, "input_cache_write": null, "output_total": 33202, "reasoning": 434, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 678072, "output_tokens": 33202, "total_tokens": 711274, "input_tokens_details": { "cached_tokens": 588624 }, "output_tokens_details": { "reasoning_tokens": 434 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-6-astra", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "astra", "repeat": "2", "authoring_model": "gpt-6-astra", "effort": "medium", "transport": "responses", "generation_id": "1469e419-cc04-4dd6-b9bb-63fc2df67f50", "target_id": "4365", "status": "completed", "excluded_reason": null, "source_file": "astra-2.json", "active_seconds": 564.5575435161591, "review_stage_seconds": null, "queue_seconds": 1553.3584852218628, "cost": { "min": 3.4422250000000005, "max": 3.6826624999999997, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The built-in end_session tool’s schema and behavior are not shown, leaving its prescribed argument names and values unverifiable." ], "prompt_review_notes": { "artifact_sha256": "f69921578e22cf470ce22f928a9723fa673846a664d8dee629844000bf5d2c41", "claim_status": "missing_reference_context", "note": "The sole blind-review deduction is that the builtin end_session schema is absent from the draft. Builtins are stored by reference. The repository contract supports the exact authored arguments reason=\"goodbye\" and session_escalated=false (builtin-contracts.json and builtin-tools.ts). This is missing review context, not an established generated-tool defect; retain the raw 15/16 score with this caveat.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts" }, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 31, "issues": [] }, "tokens": { "input_total": 707550, "input_uncached": 96175, "input_cached": 611375, "input_cache_write": null, "output_total": 37382, "reasoning": 768, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 707550, "output_tokens": 37382, "total_tokens": 744932, "input_tokens_details": { "cached_tokens": 611375 }, "output_tokens_details": { "reasoning_tokens": 768 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-6-astra", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "astra-low", "repeat": "1", "authoring_model": "gpt-6-astra", "effort": "low", "transport": "responses", "generation_id": "77c84f8b-74b0-4053-946e-dd4ab30ababe", "target_id": "4369", "status": "completed", "excluded_reason": null, "source_file": "astra-low-1.json", "active_seconds": 436.95181465148926, "review_stage_seconds": null, "queue_seconds": 1606.1260499954224, "cost": { "min": 3.028939, "max": 3.2429465, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The actual schema and behavior of builtin:end_session are not included, so the prescribed reason and session_escalated arguments cannot be confirmed as valid." ], "prompt_review_notes": { "artifact_sha256": "9062642f88d7375133d86f129278e03d3b4dc15e8b9454d2798f09d432de687f", "claim_status": "missing_reference_context", "note": "The sole blind-review deduction is that the builtin end_session schema is absent from the draft. Builtins are stored by reference. The repository contract supports the authored reason and session_escalated arguments. This is missing review context, not an established generated-tool defect; retain the raw 15/16 score with this caveat.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts" }, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 25, "issues": [] }, "tokens": { "input_total": 696562, "input_uncached": 85603, "input_cached": 610959, "input_cache_write": null, "output_total": 31239, "reasoning": 672, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 696562, "output_tokens": 31239, "total_tokens": 727801, "input_tokens_details": { "cached_tokens": 610959 }, "output_tokens_details": { "reasoning_tokens": 672 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-6-astra", "resolved_reasoning": { "context": "all_turns", "effort": "low", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "astra-low", "repeat": "2", "authoring_model": "gpt-6-astra", "effort": "low", "transport": "responses", "generation_id": "728f0cc0-bd20-47af-b74b-6b793fa82287", "target_id": "4368", "status": "completed", "excluded_reason": null, "source_file": "astra-low-2.json", "active_seconds": 385.6148955821991, "review_stage_seconds": null, "queue_seconds": 1722.3566393852234, "cost": { "min": 2.727373, "max": 2.9280354999999996, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The artifact does not provide the actual schema or implementation behavior for builtin:end_session, so alignment of reason='goodbye' and session_escalated=false cannot be confirmed without relying on unwritten platform assumptions." ], "prompt_review_notes": { "artifact_sha256": "c1953fa8b35618ea5271f9728a4a126d8816bc6883e08710dbc25f4a5d81dbf8", "claim_status": "missing_reference_context", "note": "The sole blind-review deduction is that the builtin end_session schema is absent from the draft. Builtins are stored by reference. The repository contract supports the authored reason and session_escalated arguments. This is missing review context, not an established generated-tool defect; retain the raw 15/16 score with this caveat.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts" }, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 27, "issues": [] }, "tokens": { "input_total": 625238, "input_uncached": 80265, "input_cached": 544973, "input_cache_write": null, "output_total": 27595, "reasoning": 243, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 625238, "output_tokens": 27595, "total_tokens": 652833, "input_tokens_details": { "cached_tokens": 544973 }, "output_tokens_details": { "reasoning_tokens": 243 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-6-astra", "resolved_reasoning": { "context": "all_turns", "effort": "low", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "astra-low-agents", "repeat": "1", "authoring_model": "gpt-6-astra", "effort": "low", "transport": "agents", "generation_id": "4670dc6b-1d0c-44ec-96ce-204116e2baac", "target_id": "4372", "status": "failed", "excluded_reason": "Managed session creation rejected before generation: Responses strict option leaked through shared document-tool definitions. Fixed with per-adapter catalog copies; retain as setup evidence and replace with repeat 2.", "source_file": "astra-low-agents-1.json", "active_seconds": 0.37604212760925293, "review_stage_seconds": null, "queue_seconds": 1583.0857110023499, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 0, "independent_total": 21, "prompt_review_score": null, "prompt_review_maximum": null, "prompt_review_defects": null, "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [] }, "tokens": null, "parent_tokens": null, "usage": null, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": null, "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "astra-low-agents", "repeat": "2", "authoring_model": "gpt-6-astra", "effort": "low", "transport": "agents", "generation_id": "af1a3104-0678-4f77-8663-1dc0e163ef25", "target_id": "4375", "status": "completed", "excluded_reason": null, "source_file": "astra-low-agents-2.json", "active_seconds": 432.7438085079193, "review_stage_seconds": null, "queue_seconds": 0.03856945037841797, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The end-session reference contains the unresolved text “{{TOOL:end_session}}” and no actual built-in schema is shown, leaving its stated arguments and invocation format unverifiable." ], "prompt_review_notes": { "artifact_sha256": "b2adbf9f0d3d6130e08518cdcabb0244e7002271c4e25eecdc836d5ac8325102", "claim_status": "missing_reference_context", "note": "The reviewer treats {{TOOL:end_session}} as unresolved and cannot find the builtin schema in the draft. The tool token is supported Studio prompt syntax, and builtin:end_session is stored by reference. Its repository contract supports reason and session_escalated. This is missing platform context for the reviewer, not an established generated-tool defect; the raw 15/16 score is retained.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts; shared/generation-docs/dist" }, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 37, "issues": [] }, "tokens": null, "parent_tokens": null, "usage": null, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-6-astra", "resolved_reasoning": { "effort": "low", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "opus", "repeat": "1", "authoring_model": "global.anthropic.claude-opus-5", "effort": "medium", "transport": "claude_code", "generation_id": "e5fa8d74-2318-42c0-b2a7-72b9c811160b", "target_id": "4357", "status": "cancelled", "excluded_reason": "Initial MCP metadata compatibility failure; excluded from model comparison, retained as setup evidence.", "source_file": "opus-1.json", "active_seconds": 534.6388945579529, "review_stage_seconds": null, "queue_seconds": 77.42146897315979, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 0, "independent_total": 21, "prompt_review_score": null, "prompt_review_maximum": null, "prompt_review_defects": null, "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [] }, "tokens": null, "parent_tokens": null, "usage": null, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-opus-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "opus", "repeat": "2", "authoring_model": "global.anthropic.claude-opus-5", "effort": "medium", "transport": "claude_code", "generation_id": "9025f81d-ced0-4bbc-9384-77e4da871fbe", "target_id": "4362", "status": "completed", "excluded_reason": null, "source_file": "opus-2.json", "active_seconds": 803.7551734447479, "review_stage_seconds": null, "queue_seconds": 803.7355840206146, "cost": { "min": 4.00596275, "max": 4.00596275, "basis": "Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 13, "prompt_review_maximum": 16, "prompt_review_defects": [ "The verification flow acts as an order-number existence oracle: known-order postal mismatches produce “verification_failed” and the caller is told the postal code “does not match what is on that order,” while unknown numbers produce “not_found.” This conflicts with “do not hint at whether the order exists.”", "The unsupported-request instruction says the merchant’s customer service “can” perform the listed actions, which includes issuing refunds or credits, sending labels or emails, and booking callbacks. Those real-world capabilities are not established by any tool or fixture.", "The artifact does not expose the builtin end_session schema, yet directs calls with reason and session_escalated and describes hand-back behavior; that portion of tool alignment is unverifiable." ], "prompt_review_notes": { "artifact_sha256": "20701be829c38b572a35336b94727dbf898a58ae333db17db105130494982525", "claim_status": "missing_reference_context", "note": "The reviewer could not verify the builtin end_session arguments from the draft alone. The platform contract supports reason and session_escalated; builtins are stored by reference. This resolves that schema uncertainty, but does not prove deployment-specific handoff behavior or dismiss the separate wording and verification findings.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts" }, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "cancel_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "start_return", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." } ] }, "tokens": { "input_total": 1598264, "input_uncached": 2874, "input_cached": 1388777, "input_cache_write": 206613, "output_total": 80377, "reasoning": 7202, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": null, "usage": { "input_tokens": 46, "cache_creation_input_tokens": 184021, "cache_read_input_tokens": 1272805, "output_tokens": 72129, "output_tokens_details": { "thinking_tokens": 7202 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 184021 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 22, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000888, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-opus-5": { "inputTokens": 2096, "outputTokens": 80355, "cacheReadInputTokens": 1388777, "cacheCreationInputTokens": 206613, "webSearchRequests": 0, "costUSD": 4.00507475, "contextWindow": 200000, "maxOutputTokens": 64000, "thinkingTokens": 7202, "canonicalModel": "claude-opus-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-opus-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "opus", "repeat": "3", "authoring_model": "global.anthropic.claude-opus-5", "effort": "medium", "transport": "claude_code", "generation_id": "9bd37e22-d0aa-42f7-9d9e-92c768aa6507", "target_id": "4363", "status": "completed", "excluded_reason": null, "source_file": "opus-3.json", "active_seconds": 929.6743168830872, "review_stage_seconds": null, "queue_seconds": 623.684121131897, "cost": { "min": 4.209472249999999, "max": 4.209472249999999, "basis": "Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 2, "verified": 2, "expected": 3, "supplied_tests": 16, "issues": [ { "tool": "start_return", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." } ] }, "tokens": { "input_total": 1089237, "input_uncached": 2840, "input_cached": 890316, "input_cache_write": 196081, "output_total": 101128, "reasoning": 7259, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": null, "usage": { "input_tokens": 28, "cache_creation_input_tokens": 192302, "cache_read_input_tokens": 785902, "output_tokens": 91006, "output_tokens_details": { "thinking_tokens": 7259 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 192302 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 24, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000898, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-opus-5": { "inputTokens": 2062, "outputTokens": 101104, "cacheReadInputTokens": 890316, "cacheCreationInputTokens": 196081, "webSearchRequests": 0, "costUSD": 4.20857425, "contextWindow": 200000, "maxOutputTokens": 64000, "thinkingTokens": 7259, "canonicalModel": "claude-opus-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-opus-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sol", "repeat": "1", "authoring_model": "gpt-5.6-sol", "effort": "medium", "transport": "responses", "generation_id": "5888d8f7-f23f-4862-a952-292bbd2286d3", "target_id": "4358", "status": "completed", "excluded_reason": null, "source_file": "sol-1.json", "active_seconds": 238.26785230636597, "review_stage_seconds": null, "queue_seconds": 334.94532680511475, "cost": { "min": 0.8822752, "max": 0.9600712, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "cancel_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "start_return", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." } ] }, "tokens": { "input_total": 517724, "input_uncached": 77796, "input_cached": 439928, "input_cache_write": null, "output_total": 19756, "reasoning": 2440, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 517724, "output_tokens": 19756, "total_tokens": 537480, "input_tokens_details": { "cached_tokens": 439928 }, "output_tokens_details": { "reasoning_tokens": 2440 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-sol", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sol", "repeat": "2", "authoring_model": "gpt-5.6-sol", "effort": "medium", "transport": "responses", "generation_id": "b5176fb8-3c18-4f47-a049-0ca1295fcb0d", "target_id": "4366", "status": "completed", "excluded_reason": null, "source_file": "sol-2.json", "active_seconds": 260.5711660385132, "review_stage_seconds": null, "queue_seconds": 1973.5754506587982, "cost": { "min": 1.176754, "max": 1.277926, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "The sample-tool specification is invalid. Prepare it again." }, { "tool": "cancel_order", "issue": "The sample-tool specification is invalid. Prepare it again." }, { "tool": "start_return", "issue": "The sample-tool specification is invalid. Prepare it again." } ] }, "tokens": { "input_total": 929587, "input_uncached": 101172, "input_cached": 828415, "input_cache_write": null, "output_total": 22035, "reasoning": 3095, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 929587, "output_tokens": 22035, "total_tokens": 951622, "input_tokens_details": { "cached_tokens": 828415 }, "output_tokens_details": { "reasoning_tokens": 3095 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-sol", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sol-agents", "repeat": "1", "authoring_model": "gpt-5.6-sol", "effort": "medium", "transport": "agents", "generation_id": "52d92f37-f671-4598-b8fe-0d481bf07e04", "target_id": "4354", "status": "completed", "excluded_reason": null, "source_file": "sol-agents-1.json", "active_seconds": 596.0934507846832, "review_stage_seconds": null, "queue_seconds": 0.04685497283935547, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 32, "issues": [] }, "tokens": null, "parent_tokens": null, "usage": null, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-sol", "resolved_reasoning": { "effort": "medium", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sonnet", "repeat": "1", "authoring_model": "global.anthropic.claude-sonnet-5", "effort": "medium", "transport": "claude_code", "generation_id": "91faf4cb-b14b-4d5c-a80d-d22f2186233a", "target_id": "4356", "status": "failed", "excluded_reason": "Initial MCP metadata compatibility failure; excluded from model comparison, retained as setup evidence.", "source_file": "sonnet-1.json", "active_seconds": 74.72763991355896, "review_stage_seconds": null, "queue_seconds": 4.649773120880127, "cost": { "min": 0.32762919999999995, "max": 0.32762919999999995, "basis": "Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 0, "independent_total": 21, "prompt_review_score": null, "prompt_review_maximum": null, "prompt_review_defects": null, "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [] }, "tokens": { "input_total": 894053, "input_uncached": 816, "input_cached": 842251, "input_cache_write": 50986, "output_total": 3097, "reasoning": 1067, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": null, "usage": { "input_tokens": 38, "cache_creation_input_tokens": 50986, "cache_read_input_tokens": 842251, "output_tokens": 3075, "output_tokens_details": { "thinking_tokens": 1067 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 50986 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 22, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000888, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-sonnet-5": { "inputTokens": 38, "outputTokens": 3075, "cacheReadInputTokens": 842251, "cacheCreationInputTokens": 50986, "webSearchRequests": 0, "costUSD": 0.32674119999999995, "contextWindow": 1000000, "maxOutputTokens": 64000, "thinkingTokens": 1067, "canonicalModel": "claude-sonnet-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-sonnet-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sonnet", "repeat": "2", "authoring_model": "global.anthropic.claude-sonnet-5", "effort": "medium", "transport": "claude_code", "generation_id": "1d9a6ba1-8050-47fd-af4d-d292bb0c4b2e", "target_id": "4360", "status": "completed", "excluded_reason": null, "source_file": "sonnet-2.json", "active_seconds": 664.1593654155731, "review_stage_seconds": null, "queue_seconds": 192.92820024490356, "cost": { "min": 1.2353634, "max": 1.2353634, "basis": "Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The return flow misleadingly permits a caller's item description to be sent as item_sku, but the wired tool accepts only exact SKU-RED-1 for the eligible fixture. A valid item described without its SKU will therefore be reported as ineligible." ], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "cancel_order", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." }, { "tool": "start_return", "issue": "The sample tool changed after preparation. Prepare and check the updated sample tool again, or remove its mock specification when implementing a real connection." } ] }, "tokens": { "input_total": 1471091, "input_uncached": 812, "input_cached": 1327487, "input_cache_write": 142792, "output_total": 61216, "reasoning": 2569, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": null, "usage": { "input_tokens": 32, "cache_creation_input_tokens": 142596, "cache_read_input_tokens": 1223011, "output_tokens": 61187, "output_tokens_details": { "thinking_tokens": 2569 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 142596 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 24, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000898, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-sonnet-5": { "inputTokens": 34, "outputTokens": 61192, "cacheReadInputTokens": 1327487, "cacheCreationInputTokens": 142792, "webSearchRequests": 0, "costUSD": 1.2344654000000002, "contextWindow": 1000000, "maxOutputTokens": 64000, "thinkingTokens": 2569, "canonicalModel": "claude-sonnet-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-sonnet-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sonnet", "repeat": "3", "authoring_model": "global.anthropic.claude-sonnet-5", "effort": "medium", "transport": "claude_code", "generation_id": "ccff4400-ff98-45d3-a79c-1c1c335e6426", "target_id": "4364", "status": "completed", "excluded_reason": null, "source_file": "sonnet-3.json", "active_seconds": 457.4228105545044, "review_stage_seconds": null, "queue_seconds": 1516.152621269226, "cost": { "min": 1.4268505000000002, "max": 1.4268505000000002, "basis": "Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 13, "prompt_review_maximum": 16, "prompt_review_defects": [ "The start_return scenario for ORDER-1003 with a nonmatching item SKU does not require the correct postal code and is evaluated before the wrong-postal scenario. A caller supplying ORDER-1003, any wrong postal code, and a wrong SKU can therefore learn that SKU-RED-1 is returnable, violating the no-disclosure-on-verification-failure requirement." ], "prompt_review_notes": { "artifact_sha256": "a41454db7f20fad721d9872ae6cc909164aa82d7290f0d017a606f3c053af5c9", "claim_status": "not_reproduced", "note": "The blind reviewer claimed that a wrong-item return reveals SKU-RED-1 before postal verification. Executing the unchanged saved Python with ORDER-1003, postal 99999 and item SKU-NOT-ON-ORDER returned verification_failed with no SKU; the saved source includes postal_code=10003 in the wrong-item branch. Both follow-up privacy checks pass. Preserve the raw 13/16 model score, but do not treat this alleged runtime leak as established or use it uncritically to rank Sonnet.", "evidence": "sonnet-3.evaluation.json: final two independent_checks; sonnet-3.json: start_return.config.source, 1003 verified wrong item scenario" }, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "Keep the sample specification with its generated source. Prepare this sample tool again." }, { "tool": "cancel_order", "issue": "Keep the sample specification with its generated source. Prepare this sample tool again." }, { "tool": "start_return", "issue": "Keep the sample specification with its generated source. Prepare this sample tool again." } ] }, "tokens": { "input_total": 2231987, "input_uncached": 826, "input_cached": 2065620, "input_cache_write": 165541, "output_total": 59911, "reasoning": 714, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": null, "usage": { "input_tokens": 48, "cache_creation_input_tokens": 165541, "cache_read_input_tokens": 2065620, "output_tokens": 59889, "output_tokens_details": { "thinking_tokens": 714 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 165541 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 22, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000888, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-sonnet-5": { "inputTokens": 48, "outputTokens": 59889, "cacheReadInputTokens": 2065620, "cacheCreationInputTokens": 165541, "webSearchRequests": 0, "costUSD": 1.4259625000000002, "contextWindow": 1000000, "maxOutputTokens": 64000, "thinkingTokens": 714, "canonicalModel": "claude-sonnet-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-sonnet-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "sonnet-opus", "repeat": "2", "authoring_model": "global.anthropic.claude-opus-5", "effort": "medium", "transport": "claude_code", "generation_id": "9f33f6e5-dc94-4031-b27a-c2b1422e7104", "target_id": "4370", "status": "completed", "excluded_reason": null, "source_file": "sonnet-opus-2.json", "active_seconds": 1720.3735308647156, "review_stage_seconds": 1056.2141654491425, "queue_seconds": 1638.1328864097595, "cost": { "min": 6.7287724, "max": 6.7287724, "basis": "Sum of measured first-stage and review-stage list-price estimates. Claude Code list-price estimate, including reported helper-model usage; not an AWS invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 14, "prompt_review_maximum": 16, "prompt_review_defects": [ "The lookup and action tools create an order-existence oracle: a known order number with the wrong postal code returns “verification_failed,” while an unknown number returns “not_found,” contradicting the instruction not to reveal whether the order exists.", "The actual schema and return behavior of the wired built-in end_session tool are not included, so its instructed arguments cannot be fully checked against the artifact." ], "prompt_review_notes": { "artifact_sha256": "4b7c58a9adeafa310c72c79eabe71c30e800755fb5cc5cd54e6295f8b1c0ea54", "claim_status": "missing_reference_context", "note": "The platform end_session contract supports the authored reason and session_escalated arguments; the draft stores this builtin by reference. Its schema being absent from the artifact is missing reviewer context. This does not establish that a deployment-specific handoff works, and it does not resolve the separate verification-wording finding. The raw 14/16 review is retained.", "evidence": "apps/ai-manager/lib/generation-mcp/builtin-contracts.json; apps/ai-manager/lib/generation-mcp/builtin-tools.ts; sonnet-opus-2.json" }, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 27, "issues": [] }, "tokens": { "input_total": 2710813, "input_uncached": 2880, "input_cached": 2457847, "input_cache_write": 250086, "output_total": 107624, "reasoning": 12821, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "parent_tokens": { "input_total": 1471091, "input_uncached": 812, "input_cached": 1327487, "input_cache_write": 142792, "output_total": 61216, "reasoning": 2569, "basis": "All CLI-reported models; input includes uncached, cache read and cache creation categories" }, "usage": { "input_tokens": 68, "cache_creation_input_tokens": 244895, "cache_read_input_tokens": 2299304, "output_tokens": 97602, "output_tokens_details": { "thinking_tokens": 12821 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 244895 }, "inference_geo": "", "iterations": [], "speed": "standard" }, "model_usage": { "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "inputTokens": 778, "outputTokens": 22, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.000888, "contextWindow": 200000, "maxOutputTokens": 32000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-4-5", "provider": "bedrock", "costBasis": "list" }, "global.anthropic.claude-opus-5": { "inputTokens": 2102, "outputTokens": 107602, "cacheReadInputTokens": 2457847, "cacheCreationInputTokens": 250086, "webSearchRequests": 0, "costUSD": 5.492521000000001, "contextWindow": 200000, "maxOutputTokens": 64000, "thinkingTokens": 12821, "canonicalModel": "claude-opus-5", "provider": "bedrock", "costBasis": "list" } }, "prompt_sha256": "36aeecc19d3ce39cfd2647ca76e96de16d26ea25b8e75caa341e41a1db50a7a8", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "global.anthropic.claude-opus-5", "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": { "candidate": "sonnet", "repeat": "2", "generation_id": "1d9a6ba1-8050-47fd-af4d-d292bb0c4b2e", "target_id": "4360", "draft_sha256": "f5e74b55f219c683003f3fb7ac11161f5c1683e87a3c421dc6e0b239f3c677fd" } }, { "candidate": "terra", "repeat": "1", "authoring_model": "gpt-5.6-terra", "effort": "medium", "transport": "responses", "generation_id": "d08a66d6-016c-496a-8bd3-92bbc970ed00", "target_id": "4355", "status": "completed", "excluded_reason": null, "source_file": "terra-1.json", "active_seconds": 142.32388830184937, "review_stage_seconds": null, "queue_seconds": 0.03785848617553711, "cost": { "min": 0.3422334000000001, "max": 0.36868539999999994, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 15, "prompt_review_maximum": 16, "prompt_review_defects": [ "The cancellation flow does not explicitly require lookup_order before cancel_order, even though the agent is told not to call cancel_order for shipped orders and cannot otherwise reliably know shipment state." ], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." }, { "tool": "cancel_order", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." }, { "tool": "start_return", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." } ] }, "tokens": { "input_total": 470691, "input_uncached": 52904, "input_cached": 417787, "input_cache_write": null, "output_total": 12739, "reasoning": 988, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 470691, "output_tokens": 12739, "total_tokens": 483430, "input_tokens_details": { "cached_tokens": 417787 }, "output_tokens_details": { "reasoning_tokens": 988 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-terra", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "terra", "repeat": "2", "authoring_model": "gpt-5.6-terra", "effort": "medium", "transport": "responses", "generation_id": "daab70d9-d613-4400-886e-e5ce110109a1", "target_id": "4367", "status": "completed", "excluded_reason": null, "source_file": "terra-2.json", "active_seconds": 121.04979038238525, "review_stage_seconds": null, "queue_seconds": 1395.0998022556305, "cost": { "min": 0.349737, "max": 0.3795399999999999, "basis": "Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 12, "prompt_review_maximum": 16, "prompt_review_defects": [ "The lookup fixture provides no actual estimated arrival and gives most orders only the vague location “Carrier or destination sample location.”", "Sample order numbers, matching postal codes, and the return SKU are hidden in implementation source rather than documented as caller-usable fixtures.", "The cancellation flow does not clearly require a status lookup before applying “Do NOT use for shipped orders,” even though shipped state otherwise becomes known only through lookup or the prohibited cancellation call.", "“Before every call” unintentionally requires order verification before calling end_session, conflicting with ordinary goodbye handling." ], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." }, { "tool": "cancel_order", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." }, { "tool": "start_return", "issue": "Handwritten Python; saved sample test specification absent. Independently tested in code-executor mock mode with no integration grants." } ] }, "tokens": { "input_total": 545211, "input_uncached": 59606, "input_cached": 485605, "input_cache_write": null, "output_total": 11117, "reasoning": 1710, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 545211, "output_tokens": 11117, "total_tokens": 556328, "input_tokens_details": { "cached_tokens": 485605 }, "output_tokens_details": { "reasoning_tokens": 1710 } }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-terra", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "terra-agents", "repeat": "1", "authoring_model": "gpt-5.6-terra", "effort": "medium", "transport": "agents", "generation_id": "1319c785-72a5-4d16-b196-57321faf919e", "target_id": "4371", "status": "failed", "excluded_reason": "Managed session creation rejected before generation: Responses strict option leaked through shared document-tool definitions. Fixed with per-adapter catalog copies; retain as setup evidence and replace with repeat 2.", "source_file": "terra-agents-1.json", "active_seconds": 1.0571329593658447, "review_stage_seconds": null, "queue_seconds": 1582.0315651893616, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 0, "independent_total": 21, "prompt_review_score": null, "prompt_review_maximum": null, "prompt_review_defects": null, "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [] }, "tokens": null, "parent_tokens": null, "usage": null, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": null, "resolved_reasoning": null, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "terra-agents", "repeat": "2", "authoring_model": "gpt-5.6-terra", "effort": "medium", "transport": "agents", "generation_id": "b6c60f3e-3a83-409d-97e5-f737d3f7459e", "target_id": "4374", "status": "completed", "excluded_reason": null, "source_file": "terra-agents-2.json", "active_seconds": 622.9496810436249, "review_stage_seconds": null, "queue_seconds": 0.04198026657104492, "cost": { "min": null, "max": null, "basis": "Provider usage unavailable; not zero" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 3, "verified": 3, "expected": 3, "supplied_tests": 14, "issues": [] }, "tokens": { "input_total": 1778046, "input_uncached": 91174, "input_cached": 1686872, "input_cache_write": null, "output_total": 16759, "reasoning": 1622, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": null, "usage": { "input_tokens": 1778046, "input_tokens_details": { "cached_tokens": 1686872 }, "output_tokens": 16759, "output_tokens_details": { "reasoning_tokens": 1622 }, "total_tokens": 1794805 }, "model_usage": null, "prompt_sha256": "db084657f70655b6c4e0d8c2b9519e70ba07039776918b61bf78fbcb61b9380d", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-terra", "resolved_reasoning": { "effort": "medium", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": null }, { "candidate": "terra-sol", "repeat": "1", "authoring_model": "gpt-5.6-sol", "effort": "medium", "transport": "responses", "generation_id": "ff1fe615-8df1-4f46-ba86-38f175781bca", "target_id": "4361", "status": "completed", "excluded_reason": null, "source_file": "terra-sol-1.json", "active_seconds": 433.36543250083923, "review_stage_seconds": 291.04154419898987, "queue_seconds": 548.1245412826538, "cost": { "min": 1.5490977999999997, "max": 1.6852958000000005, "basis": "Sum of measured first-stage and review-stage list-price estimates. Recorded Responses tokens × standard list rates; range assumes zero through all uncached input charged at cache-write premium; not an invoice" }, "independent_passed": 21, "independent_total": 21, "prompt_review_score": 16, "prompt_review_maximum": 16, "prompt_review_defects": [], "prompt_review_notes": null, "saved_sample_specs": { "intact": 0, "verified": 0, "expected": 3, "supplied_tests": 0, "issues": [ { "tool": "lookup_order", "issue": "The sample-tool specification is invalid. Prepare it again." }, { "tool": "cancel_order", "issue": "The sample-tool specification is invalid. Prepare it again." }, { "tool": "start_return", "issue": "The sample-tool specification is invalid. Prepare it again." } ] }, "tokens": { "input_total": 969147, "input_uncached": 109746, "input_cached": 859401, "input_cache_write": null, "output_total": 21206, "reasoning": 2583, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "parent_tokens": { "input_total": 470691, "input_uncached": 52904, "input_cached": 417787, "input_cache_write": null, "output_total": 12739, "reasoning": 988, "basis": "OpenAI input already includes cached input; output already includes reasoning; cache-write count unavailable" }, "usage": { "input_tokens": 969147, "output_tokens": 21206, "total_tokens": 990353, "input_tokens_details": { "cached_tokens": 859401 }, "output_tokens_details": { "reasoning_tokens": 2583 } }, "model_usage": null, "prompt_sha256": "36aeecc19d3ce39cfd2647ca76e96de16d26ea25b8e75caa341e41a1db50a7a8", "docs_sha256": "fe3a5796fecbfa39bd1e0f4f3deeb39aa41a4e235d8fbd54733ada0f7229beec", "resolved_model": "gpt-5.6-sol", "resolved_reasoning": { "context": "all_turns", "effort": "medium", "mode": "standard", "summary": null }, "runtime_models": [ "gpt-realtime-2" ], "parent": { "candidate": "terra", "repeat": "1", "generation_id": "d08a66d6-016c-496a-8bd3-92bbc970ed00", "target_id": "4355", "draft_sha256": "83531796385455b6a36da8d173c239b12f6b4b9854cd0af25c25ee6192d807ac" } } ] }