{ "schemaVersion": 1, "sourceRevision": "720c2a48c3d7599ec2d987cdff0283e5fea1d83b", "status": "acceptance_definition_not_execution_results", "profiles": { "insurance-fnol": {"dataset": "fnol-v1", "vanguardAgentId": "4546"}, "delta-original": {"dataset": "delta-airline-v1", "vanguardAgentId": "4547", "sourceBaseline": "warehouse_not_populated"}, "netomi-delta": {"dataset": "netomi-delta-v1", "vanguardAgentId": "4548", "sourceBaseline": "warehouse_not_populated"}, "deloitte": {"dataset": "cciq-demo-v7", "vanguardAgentId": "4549"}, "airline-delta": {"dataset": "cciq-demo-v7", "vanguardAgentId": "4550"}, "airline-flightstar": {"dataset": "cciq-demo-v7", "vanguardAgentId": "4551"} }, "responseShapes": ["metric", "ranking", "comparison", "diagnostic", "narrative", "record_listing", "artifact", "definition"], "chartTypes": ["line", "area", "bar", "horizontal_bar", "donut", "composed", "scatter", "radial", "treemap", "table"], "cases": [ { "id": "dataset-count", "profiles": ["insurance-fnol", "delta-original", "netomi-delta", "deloitte", "airline-delta", "airline-flightstar"], "mode": "chat", "shape": "metric", "sourceTools": ["query_data_lake", "make_stat_tile"], "turns": ["How many contacts are in this dataset?"], "verify": ["Exact distinct contacts from the pinned source manifest", "No evidence-sample denominator", "Appropriate compact answer"] }, { "id": "dataset-story", "profiles": ["insurance-fnol", "delta-original", "netomi-delta", "deloitte", "airline-delta", "airline-flightstar"], "mode": "chat", "shape": "narrative", "sourceTools": ["list_memory_pages", "read_memory_page"], "turns": ["What's happening in this data?"], "verify": ["Compiled dataset narrative where available", "Correct domain vocabulary and substantive findings", "Missing compilation is explicit and remains a parity gap"] }, { "id": "weekly-story", "profiles": ["deloitte", "airline-delta", "airline-flightstar"], "mode": "chat", "shape": "narrative", "sourceTools": ["list_trends", "read_trend"], "turns": ["What's the state of the business this week?"], "verify": ["Reference-date window and appropriate report version", "Equivalent findings, comparisons, recommendations and follow-ups", "No silent switch to dataset-wide counts"] }, { "id": "driver-ranking", "profiles": ["deloitte", "airline-delta", "airline-flightstar"], "mode": "chat", "shape": "ranking", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["What are our top drivers?", "What are people complaining about?"], "verify": ["Literal numeric ranking without skipping routine-sounding categories", "Driver counts distinct from secondary mentions or friction labels", "Chart, selected focus and local actions"] }, { "id": "baggage-week-comparison", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["get_intent_taxonomy", "query_data_lake", "make_metric_strip", "make_chart"], "turns": ["How's baggage doing this week compared to last?"], "verify": ["Exact count/rate/window and prior-period delta", "Fresh authoritative metrics rather than illustrative narrative numbers", "No duplicated metric strip and chart facts"] }, { "id": "recorded-policy-follow-ups", "profiles": ["airline-delta"], "mode": "chat", "shape": "ranking", "sourceTools": ["query_data_lake", "make_chart", "make_stat_tile"], "turns": ["Break down policy confusion by hub", "Break down Atlanta policy calls", "How are we doing on AHT?"], "verify": ["Policy confusion retains its friction definition during drill-down", "Hub/category selection retains applicable window and release", "AHT topic change does not silently inherit unrelated policy filters", "Recorded component order, focus highlights and local chips"] }, { "id": "aht-comparison", "profiles": ["airline-delta"], "mode": "chat", "shape": "metric", "sourceTools": ["query_data_lake", "make_stat_tile", "make_chart"], "turns": ["How are we doing on AHT?", "Compare Atlanta to Minneapolis"], "verify": ["Operational AHT with correct weighting and baseline", "Seconds and minutes never mixed", "Scope retained across the comparison"] }, { "id": "escalation-by-hub", "profiles": ["airline-delta"], "mode": "chat", "shape": "ranking", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["Break down escalations by hub"], "verify": ["Operational escalation flag and denominator are explicit", "Source disposition inconsistency recorded rather than hidden", "Aggregate chart instead of a record table"] }, { "id": "disposition-share", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["What's the disposition breakdown this week?"], "verify": ["Full distribution, share denominator and Other grouping", "Donut or appropriate equivalent share visual"] }, { "id": "longest-transfers", "profiles": ["airline-delta"], "mode": "chat", "shape": "record_listing", "sourceTools": ["query_data_lake", "get_contact"], "turns": ["Top 10 longest transferred calls"], "verify": ["Ranking by requested handle time", "Contact ID, timestamp, customer, hub, intent, disposition and one metric unit", "Eight-row preview then Show all", "Every contact row opens the correct transcript"] }, { "id": "themes-to-plural-examples", "profiles": ["airline-delta"], "mode": "chat", "shape": "narrative", "sourceTools": ["search_transcripts", "get_contact"], "turns": ["What are customers saying about vouchers?", "Show a few representative calls"], "verify": ["Themes first without dumping transcripts", "Follow-up uses prior result IDs and ordering", "Up to five distinct contact cards and every fetched card visible"] }, { "id": "singular-example", "profiles": ["airline-delta"], "mode": "chat", "shape": "narrative", "sourceTools": ["search_transcripts", "get_contact"], "turns": ["Give me one representative baggage call"], "verify": ["Exactly one conversation", "Summary and selected example relevance", "No additional contact shopping or unrelated aggregate sections"] }, { "id": "filtered-search", "profiles": ["airline-delta"], "mode": "chat", "shape": "narrative", "sourceTools": ["get_intent_taxonomy", "search_transcripts", "get_contact"], "turns": ["Find baggage conversations in Atlanta this week", "Show the call"], "verify": ["Intent/hub/date/annotation filters applied before final ranking", "Selected evidence belongs to the requested slice", "No stale or cross-dataset hits"] }, { "id": "fnol-claim-flow", "profiles": ["insurance-fnol"], "mode": "chat", "shape": "narrative", "sourceTools": ["get_intent_taxonomy", "query_data_lake", "search_transcripts", "get_contact"], "turns": ["What claim types are represented?", "Show me an accident claim", "What happened on that call?"], "verify": ["Flat insurance taxonomy works", "No airline terminology", "One claim remains one interaction", "No statistical claims from two examples"] }, { "id": "taxonomy-and-goals", "profiles": ["insurance-fnol", "airline-delta"], "mode": "chat", "shape": "definition", "sourceTools": ["get_intent_taxonomy", "query_data_lake"], "turns": ["What call categories are available?", "Which categories have containment below their goal?"], "verify": ["Profile-specific tree or flat taxonomy", "Goal is a configured target, not an observed metric", "Fuzzy category wording resolves to correct leaves"] }, { "id": "reported-observed-intent", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["detect_intent_drift", "query_data_lake", "get_contact"], "turns": ["Where do reported and observed intents disagree?", "Show calls behind the largest mismatch"], "verify": ["Mismatch pairs, direction, count and version eligibility", "Separate reported and observed taxonomy breadcrumbs", "Disagreement is not automatically a classification error"] }, { "id": "latest-executive-summary", "profiles": ["insurance-fnol", "airline-delta"], "mode": "chat", "shape": "artifact", "sourceTools": ["list_memory_pages", "read_memory_page"], "turns": ["Show me the latest executive summary"], "verify": ["Correct active artifact with version, KPIs and citations", "One framing sentence and report card without duplicate narrative"] }, { "id": "hour-day-week-reports", "profiles": ["airline-delta"], "mode": "chat", "shape": "artifact", "sourceTools": ["list_trends", "read_trend"], "turns": ["Show me the latest weekly trend report", "Show the latest daily report", "Show the report for June 2, 2026 from 12:00 to 13:00 UTC"], "verify": ["All three tiers and correct explicit windows", "Driver chart visible; root causes, friction and escalation collapsed", "Pinned artifact versions and cited metric scopes"] }, { "id": "citation-navigation", "profiles": ["airline-delta"], "mode": "browser", "shape": "artifact", "sourceTools": ["list_citations", "resolve_citation", "get_contact", "read_trend", "read_memory_page"], "turns": ["Show me the latest weekly trend report"], "verify": ["Inline citation opens contextual contact picker", "Driver, root-cause, friction and escalation resolvers preserve window and ranking", "Contact opens right drawer with transcript and metadata", "Back navigation restores prior scroll and expansion"] }, { "id": "reverse-citations", "profiles": ["airline-delta"], "mode": "chat", "shape": "artifact", "sourceTools": ["get_contact", "list_citations", "read_trend", "read_memory_page"], "turns": ["Show me Maya's call", "Which reports cite this contact?"], "verify": ["Use actual evidence edges, not inferred report membership", "Missing direct citation edges produce an honest empty result"] }, { "id": "peer-anomalies", "profiles": ["airline-delta"], "mode": "chat", "shape": "diagnostic", "sourceTools": ["detect_anomalies", "query_data_lake", "make_chart"], "turns": ["Which hubs have unusual AHT changes?", "What about containment?", "What about escalation?", "What about repeat calls?", "What about CSAT?", "What about volume?"], "verify": ["All six detector metrics", "Recent/baseline windows, population and peer-group method", "Change-point evidence and sparse-baseline handling", "Normalize unequal exposure for volume; document corrected source behavior"] }, { "id": "internal-change-diagnostic", "profiles": ["airline-delta"], "mode": "chat", "shape": "diagnostic", "sourceTools": ["detect_anomalies", "query_data_lake", "search_transcripts", "make_chart"], "turns": ["Why did Atlanta handle time increase?"], "verify": ["Metric shift joined to internal policy/system/staffing changes", "Dates and scope agree", "Correlation framed as a supported hypothesis", "Data from outside transcripts is available"] }, { "id": "external-disruption-diagnostic", "profiles": ["airline-delta"], "mode": "chat", "shape": "diagnostic", "sourceTools": ["detect_anomalies", "query_data_lake", "search_transcripts", "make_chart"], "turns": ["Which flight disruptions explain the call-volume spikes?"], "verify": ["External disruption events joined by applicable hub/date/flight", "Weather, ATC and mechanical context distinct from internal changes", "No unsupported causal certainty"] }, { "id": "survey-metrics", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_metric_strip", "make_chart"], "turns": ["Compare CSAT, top-two-box satisfaction and NPS by hub"], "verify": ["Respondent denominators and duplicate-join protection", "NPS promoter-minus-detractor definition", "Missing surveys excluded rather than treated as zero", "No substitution of transcript sentiment"] }, { "id": "customer-and-queue-cohorts", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["Compare repeat-call rates by customer tier", "Break that down by queue and channel"], "verify": ["Customer, queue, channel and agent fields remain queryable", "Correct distinct-contact joins", "Cohort and rate definition retained across turns"] }, { "id": "reservation-outcomes", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["Did rebooked customers ultimately fly?"], "verify": ["Reservation/travel outcomes joined to contacts/customers without multiplication", "Actual travel outcome distinct from annotation resolution"] }, { "id": "proactive-outreach", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["Which proactive messages were followed by a call within 24 hours?"], "verify": ["Proactive-message data and correct message/customer denominator", "No invented causal deflection percentage"] }, { "id": "whatif-ai-routing", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["get_intent_taxonomy", "run_whatif"], "turns": ["What if we routed same-day flight changes to AI first?"], "verify": ["route_intent_to_ai behavior", "Current blended versus historical AI rate", "Explicit capacity/mix/quality assumptions and directional estimate", "Active dataset scope"] }, { "id": "whatif-auth-step", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["run_whatif"], "turns": ["What if we removed Atlanta's second authentication step?"], "verify": ["remove_second_auth behavior", "Correct event and before/after AHT", "Assumption attributing the lift is explicit, not a proven causal effect"] }, { "id": "whatif-billing-deflection", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["run_whatif"], "turns": ["What if we deflected 10% of human-handled billing calls?"], "verify": ["deflect_billing_pct behavior", "Configured cost rate, 30-day volume, AHT, time and savings arithmetic", "No guaranteed outcome or automatic action"] }, { "id": "whatif-containment", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["get_intent_taxonomy", "run_whatif"], "turns": ["What if containment improved by 5 percentage points for same-day changes?"], "verify": ["containment_uplift behavior", "Percentage points distinct from percent increase", "Configured cap and population", "Unsupported scenarios rejected explicitly"] }, { "id": "prescriptive-priorities", "profiles": ["airline-delta"], "mode": "chat", "shape": "diagnostic", "sourceTools": ["list_trends", "read_trend", "list_memory_pages", "read_memory_page", "get_intent_taxonomy", "query_data_lake"], "turns": ["Where should we focus to reduce avoidable calls?", "What would change that recommendation?"], "verify": ["Diagnose before recommending", "At most three grounded moves with scope, owner and measurable outcome", "No invented financial benefit", "Local recommendations and follow-ups meet length limits"] }, { "id": "nonstandard-window", "profiles": ["airline-delta"], "mode": "chat", "shape": "comparison", "sourceTools": ["query_data_lake", "make_chart"], "turns": ["Compare baggage calls from May 30 through June 2, 2026 with the preceding four days"], "verify": ["Arbitrary supported window and adjacent baseline", "No substitution of nearest weekly artifact", "Explicit timezone and inclusive/exclusive boundaries"] }, { "id": "no-data-and-errors", "profiles": ["insurance-fnol", "airline-delta"], "mode": "failure_injection", "shape": "metric", "sourceTools": ["query_data_lake", "search_transcripts", "list_trends"], "turns": ["How many calls are in a period outside this dataset?"], "verify": ["Successful zero rows distinct from unavailable warehouse", "Query validation/error can be repaired within bounded tool rounds", "Missing report falls back to eligible facts without invented narrative", "Partial streams, tool exhaustion and provider failures are visible"] }, { "id": "recorded-landing-journey", "profiles": ["airline-delta"], "mode": "browser", "shape": "narrative", "sourceTools": ["list_trends", "read_trend", "make_stat_tile", "make_metric_strip", "make_chart"], "turns": ["Break down policy confusion by hub", "Break down Atlanta policy calls", "How are we doing on AHT?", "Show me the latest weekly trend report"], "verify": ["Agent library then two-column briefing", "Briefing replaced by chat on first question", "Clicked metric/window/release carried into chat", "Full callsight-demo-v2.mov journey including nested evidence"] }, { "id": "all-visual-variants", "profiles": ["airline-delta"], "mode": "fixture_replay", "shape": "comparison", "sourceTools": ["make_chart", "make_stat_tile", "make_metric_strip", "get_contact", "read_trend", "read_memory_page", "resolve_citation"], "turns": [], "chartTypes": ["line", "area", "bar", "horizontal_bar", "donut", "composed", "scatter", "radial", "treemap", "table"], "verify": ["Same payload in original and catalogue renderer", "All chart types, hero and strip, table, contact, picker, summary and trend", "Message length, order, width, labels, units, highlights, expanders, drawers and follow-up positions", "Historical messages remain interactive"] }, { "id": "configurable-processing", "profiles": ["insurance-fnol", "airline-delta"], "mode": "compiler", "shape": "artifact", "sourceTools": [], "turns": [], "verify": ["Annotation/taxonomy/evidence parity", "Hour/day/week compilation and alternate configured hierarchy", "Child coverage, late corrections and dependency invalidation", "Separate analysis windows and briefing cadence", "Rebuild and published-version reproducibility"] }, { "id": "studio-access-and-replay", "profiles": ["insurance-fnol", "delta-original", "netomi-delta", "deloitte", "airline-delta", "airline-flightstar"], "mode": "browser", "shape": "narrative", "sourceTools": [], "turns": ["What's happening in this data?"], "verify": ["Preview and published host use the same answer renderer", "Reload preserves answers, attachments and evidence versions", "SSO, tenant switch, audience revocation and feature gate", "Brand configuration never rewrites source quotations", "No cross-profile, cross-user or cross-tenant result reuse"] } ] }