From 4206b651f4d36306649a4f31a9faf040f224323f Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 3 Jun 2026 11:57:36 -0400 Subject: [PATCH 1/2] Fix BidCliq compliance suite fixtures --- .../scenarios/invalid_transitions.yaml | 5 + .../domains/media-buy/state-machine.yaml | 18 +- .../scenarios/invalid_transitions.yaml | 5 + .../protocols/media-buy/state-machine.yaml | 18 +- .../pagination-integrity-list-accounts.yaml | 254 +++++++++++------- .../3.0.14/universal/storyboard-schema.yaml | 16 +- .../domains/media-buy/state-machine.yaml | 6 +- .../protocols/media-buy/state-machine.yaml | 6 +- .../pagination-integrity-list-accounts.yaml | 193 ++++++++----- .../universal/storyboard-schema.yaml | 15 +- .../comply-test-controller-request.json | 53 ++-- scripts/lint-storyboard-contradictions.cjs | 1 + server/src/training-agent/account-handlers.ts | 78 +++++- .../training-agent/comply-test-controller.ts | 6 +- server/src/training-agent/task-handlers.ts | 1 + server/src/training-agent/tenants/router.ts | 1 + .../tenants/tenant-smoke.test.ts | 1 + .../tests/unit/comply-test-controller.test.ts | 63 ++++- server/tests/unit/training-agent.test.ts | 1 + .../protocols/media-buy/state-machine.yaml | 6 +- .../pagination-integrity-list-accounts.yaml | 193 ++++++++----- .../source/universal/storyboard-schema.yaml | 15 +- .../comply-test-controller-request.json | 27 +- 23 files changed, 697 insertions(+), 285 deletions(-) diff --git a/dist/compliance/3.0.14/domains/media-buy/scenarios/invalid_transitions.yaml b/dist/compliance/3.0.14/domains/media-buy/scenarios/invalid_transitions.yaml index 112336abc5..d90432f87a 100644 --- a/dist/compliance/3.0.14/domains/media-buy/scenarios/invalid_transitions.yaml +++ b/dist/compliance/3.0.14/domains/media-buy/scenarios/invalid_transitions.yaml @@ -109,6 +109,8 @@ phases: sample_request: buying_mode: "brief" brief: "Display inventory on outdoor lifestyle content. Q3 flight." + filters: + is_fixed_price: true account: brand: domain: "acmeoutdoor.example" @@ -121,6 +123,9 @@ phases: validations: - check: response_schema description: "Response matches get-products-response.json schema" + - check: field_present + path: "products[0].pricing_options[0].fixed_price" + description: "The captured pricing option is fixed-price; fixed-price storyboards do not send bid_price" - id: create_buy title: "Create a buy for the error probes" diff --git a/dist/compliance/3.0.14/domains/media-buy/state-machine.yaml b/dist/compliance/3.0.14/domains/media-buy/state-machine.yaml index e15bacab5c..ce0c9d4093 100644 --- a/dist/compliance/3.0.14/domains/media-buy/state-machine.yaml +++ b/dist/compliance/3.0.14/domains/media-buy/state-machine.yaml @@ -157,12 +157,12 @@ phases: operator: "pinnacle-agency.example" brand: domain: "acmeoutdoor.example" - start_time: "2026-07-01T00:00:00Z" + start_time: "asap" end_time: "2026-09-30T23:59:59Z" packages: - - product_id: "test-product" + - product_id: "$context.product_id" budget: 10000 - pricing_option_id: "test-pricing" + pricing_option_id: "$context.pricing_option_id" idempotency_key: "$generate:uuid_v4#media_buy_state_machine_setup_create_buy" context: @@ -222,8 +222,8 @@ phases: correlation_id: "media_buy_state_machine--pause_buy" validations: - check: field_present - path: "status" - description: "Response includes updated status" + path: "media_buy_status" + description: "Response includes updated media_buy_status" - check: field_present path: "context" @@ -258,8 +258,8 @@ phases: correlation_id: "media_buy_state_machine--resume_buy" validations: - check: field_present - path: "status" - description: "Response includes updated status" + path: "media_buy_status" + description: "Response includes updated media_buy_status" - check: field_present path: "context" @@ -295,8 +295,8 @@ phases: correlation_id: "media_buy_state_machine--cancel_buy" validations: - check: field_present - path: "status" - description: "Response includes canceled status" + path: "media_buy_status" + description: "Response includes canceled media_buy_status" - check: field_present path: "context" diff --git a/dist/compliance/3.0.14/protocols/media-buy/scenarios/invalid_transitions.yaml b/dist/compliance/3.0.14/protocols/media-buy/scenarios/invalid_transitions.yaml index 112336abc5..d90432f87a 100644 --- a/dist/compliance/3.0.14/protocols/media-buy/scenarios/invalid_transitions.yaml +++ b/dist/compliance/3.0.14/protocols/media-buy/scenarios/invalid_transitions.yaml @@ -109,6 +109,8 @@ phases: sample_request: buying_mode: "brief" brief: "Display inventory on outdoor lifestyle content. Q3 flight." + filters: + is_fixed_price: true account: brand: domain: "acmeoutdoor.example" @@ -121,6 +123,9 @@ phases: validations: - check: response_schema description: "Response matches get-products-response.json schema" + - check: field_present + path: "products[0].pricing_options[0].fixed_price" + description: "The captured pricing option is fixed-price; fixed-price storyboards do not send bid_price" - id: create_buy title: "Create a buy for the error probes" diff --git a/dist/compliance/3.0.14/protocols/media-buy/state-machine.yaml b/dist/compliance/3.0.14/protocols/media-buy/state-machine.yaml index e15bacab5c..ce0c9d4093 100644 --- a/dist/compliance/3.0.14/protocols/media-buy/state-machine.yaml +++ b/dist/compliance/3.0.14/protocols/media-buy/state-machine.yaml @@ -157,12 +157,12 @@ phases: operator: "pinnacle-agency.example" brand: domain: "acmeoutdoor.example" - start_time: "2026-07-01T00:00:00Z" + start_time: "asap" end_time: "2026-09-30T23:59:59Z" packages: - - product_id: "test-product" + - product_id: "$context.product_id" budget: 10000 - pricing_option_id: "test-pricing" + pricing_option_id: "$context.pricing_option_id" idempotency_key: "$generate:uuid_v4#media_buy_state_machine_setup_create_buy" context: @@ -222,8 +222,8 @@ phases: correlation_id: "media_buy_state_machine--pause_buy" validations: - check: field_present - path: "status" - description: "Response includes updated status" + path: "media_buy_status" + description: "Response includes updated media_buy_status" - check: field_present path: "context" @@ -258,8 +258,8 @@ phases: correlation_id: "media_buy_state_machine--resume_buy" validations: - check: field_present - path: "status" - description: "Response includes updated status" + path: "media_buy_status" + description: "Response includes updated media_buy_status" - check: field_present path: "context" @@ -295,8 +295,8 @@ phases: correlation_id: "media_buy_state_machine--cancel_buy" validations: - check: field_present - path: "status" - description: "Response includes canceled status" + path: "media_buy_status" + description: "Response includes canceled media_buy_status" - check: field_present path: "context" diff --git a/dist/compliance/3.0.14/universal/pagination-integrity-list-accounts.yaml b/dist/compliance/3.0.14/universal/pagination-integrity-list-accounts.yaml index 0c8298684e..66701f1a71 100644 --- a/dist/compliance/3.0.14/universal/pagination-integrity-list-accounts.yaml +++ b/dist/compliance/3.0.14/universal/pagination-integrity-list-accounts.yaml @@ -1,8 +1,8 @@ id: pagination_integrity_list_accounts version: "1.0.0" -title: "Pagination cursor integrity — list_accounts" +title: "Pagination continuation integrity — list_accounts" category: schema_validation -summary: "Validates the cursor↔has_more invariant by walking a paginated list_accounts response from a continuation page to a terminal page." +summary: "Validates list_accounts pagination by seeding sandbox accounts, requesting a small first page, and following its cursor once." track: core required_tools: - list_accounts @@ -16,21 +16,17 @@ narrative: | The invariant is documented in prose on `static/schemas/source/core/pagination-response.json` but JSON Schema does - not gate the two fields against each other. This storyboard exercises both - sides of the invariant on `list_accounts` in a single run. + not gate the two fields against each other. This storyboard exercises the + continuation side on `list_accounts`: when `has_more=true`, `cursor` must be + present and usable. - The runner bootstraps three accounts via `sync_accounts`, then lists them - back with `max_results=2`. The first page MUST cap at two accounts and - signal continuation with `has_more=true` plus a cursor. The runner then - follows that cursor; the next page returns the remaining account and MUST - signal `has_more=false` with no cursor. An agent that hides the third - account by reporting `has_more=false` on the first page fails the - first-page assertion. An agent that carries a stale cursor onto the - terminal page fails the second-page assertion. - - The `total_count` field is allowed to be absent (some backends cannot - compute it cheaply per the pagination-response schema) but when an agent - volunteers it, the value MUST match the seeded account count of three. + The runner seeds three sandbox accounts via `comply_test_controller`, then + lists sandbox accounts back with `max_results=2`. The first page MUST cap at + two accounts and signal continuation with `has_more=true` plus a cursor. The + runner then follows that cursor to prove the continuation token is usable. + The seed_account contract is session-scoped: seeded accounts are visible to + the requesting compliance principal for this run and MUST NOT rely on + production tenant data already present in the seller's sandbox. agent: interaction_model: stateful_preloaded @@ -44,18 +40,38 @@ caller: prerequisites: description: | - The storyboard bootstraps three accounts in the setup phase via - `sync_accounts`. The seed-DAG documented at - `static/compliance/source/universal/storyboard-schema.yaml` does not - include accounts (today's seed kinds are products, pricing_options, - creatives, plans, media_buys), so this storyboard deliberately - departs from the `controller_seeding: true` precedent used by - `pagination-integrity.yaml` (list_creatives) and bootstraps via - real protocol calls. Adding `seed_account` to the controller is - tracked separately; until then the create-then-list pattern keeps - the storyboard self-contained without extending the seed taxonomy. + The runner seeds three accounts through comply_test_controller.seed_account + so the pagination walk uses compliance-session state rather than assuming + pre-existing tenant fixtures. test_kit: "test-kits/acme-outdoor.yaml" - controller_seeding: false + controller_seeding: true + +fixtures: + accounts: + - account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" phases: - id: capability_discovery @@ -96,75 +112,126 @@ phases: value: "pagination_integrity_list_accounts--get_capabilities" description: "Context correlation_id returned unchanged" - - id: setup - title: "Bootstrap three test accounts" + - id: seed_accounts + title: "Seed three test accounts" narrative: | - Sync three sandbox accounts so the pagination walk has a - deterministic set of three items to page through with max_results=2. + Seed three sandbox accounts so the pagination walk has at least three + sandbox items to page through with max_results=2. seed_account is + idempotent within a run: the same account_id plus same fixture succeeds, + while the same account_id with a different fixture is invalid state. steps: - - id: sync_three_accounts - title: "Sync three sandbox accounts" - narrative: | - Create three sandbox accounts — Acme c/o Pinnacle, Nova c/o Pinnacle, - and Pinnacle Direct. The agent MUST accept all three and respond with - account_id values that subsequent list_accounts calls surface. - task: sync_accounts - schema_ref: "account/sync-accounts-request.json" - response_schema_ref: "account/sync-accounts-response.json" - doc_ref: "/accounts/task-reference/sync_accounts" - comply_scenario: pagination_integrity_list_accounts + - id: seed_account_1 + title: "Seed first sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account stateful: true expected: | - Accept all three accounts and return account_id for each. - + Return success: true. sample_request: - accounts: - - brand: - domain: acme-corp.com - operator: pinnacle-media.com - billing: operator - sandbox: true - - brand: - domain: nova-brands.com - operator: pinnacle-media.com - billing: operator + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_1" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true - - brand: - domain: pinnacle-media.com - operator: pinnacle-media.com - billing: operator + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_1" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" + + - id: seed_account_2 + title: "Seed second sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_2" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_2" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - idempotency_key: "$generate:uuid_v4#pagination_integrity_list_accounts_setup_sync_three_accounts" + - id: seed_account_3 + title: "Seed third sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_3" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" context: - correlation_id: "pagination_integrity_list_accounts--setup" + correlation_id: "pagination_integrity_list_accounts--seed_3" validations: - - check: response_schema - description: "Response matches sync-accounts-response.json schema" - - check: field_present - path: "accounts" - description: "Response contains accounts array" + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - id: pagination_walk title: "Walk pages with a small max_results" narrative: | - With three accounts seeded and `max_results: 2`, the first call MUST - return a continuation page (has_more=true with a cursor). The runner - then follows the cursor; the second call MUST return the terminal - page (has_more=false with no cursor). + With at least three sandbox accounts and `max_results: 2`, the first call + MUST return a continuation page (has_more=true with a cursor). The + runner then follows the cursor and validates the next page response shape. steps: - id: first_page title: "Request the first page with max_results=2" narrative: | - The buyer asks for up to two accounts. The agent has three matching - accounts, so the request is non-terminal: callers expect + The buyer asks for up to two sandbox accounts. The compliance session + has at least three seeded accounts, so the request is non-terminal: callers expect `has_more=true` and a `cursor` they can follow to retrieve the remaining account. The runner captures `pagination.cursor` into `$context.next_cursor` for the follow-up step. If the agent reports `has_more=false` on - this page (hiding the seeded third account behind a dishonest + this page (hiding the seeded third sandbox account behind a dishonest terminal signal), the field_value check on `pagination.has_more` fails — surfacing exactly the bug class this storyboard exists to catch. @@ -183,6 +250,7 @@ phases: - pagination.cursor present (an opaque continuation token) sample_request: + sandbox: true pagination: max_results: 2 @@ -194,15 +262,10 @@ phases: - check: field_value path: "pagination.has_more" value: true - description: "Three accounts seeded with max_results=2 → first page is non-terminal" + description: "At least three sandbox accounts exist with max_results=2 → first page is non-terminal" - check: field_present path: "pagination.cursor" description: "has_more=true requires a cursor — without one the caller cannot continue" - - check: field_value_or_absent - path: "pagination.total_count" - allowed_values: [3] - description: "If volunteered, total_count MUST equal the seeded set size — under-reporting hides accounts the same way a dishonest has_more does" - - check: field_present path: "context" description: "Response echoes back the context object" @@ -211,13 +274,13 @@ phases: value: "pagination_integrity_list_accounts--first_page" description: "Context correlation_id returned unchanged" - - id: terminal_page - title: "Follow the cursor to the terminal page" + - id: next_page + title: "Follow the cursor to the next page" narrative: | - The buyer follows the captured cursor. With at most one remaining - account the agent MUST return a terminal page: `has_more=false` - with no `cursor` field. A cursor on `has_more=false` is a stale - token that invites callers to follow it past the end of results. + The buyer follows the captured cursor. The response MUST remain + schema-valid and echo context. The runner does not require this page + to be terminal because sellers may already have more than the three + sandbox accounts created during setup. task: list_accounts schema_ref: "account/list-accounts-request.json" response_schema_ref: "account/list-accounts-response.json" @@ -225,38 +288,27 @@ phases: comply_scenario: pagination_integrity_list_accounts stateful: true expected: | - Return the remaining account(s) with: - - pagination.has_more = false - - pagination.cursor absent (a stale cursor on a terminal page is a - dishonest pagination signal) + Return the next page of sandbox accounts with a valid pagination block. sample_request: + sandbox: true pagination: cursor: "$context.next_cursor" max_results: 2 context: - correlation_id: "pagination_integrity_list_accounts--terminal_page" + correlation_id: "pagination_integrity_list_accounts--next_page" validations: - check: response_schema description: "Response matches list-accounts-response.json schema" - - check: field_value + - check: field_present path: "pagination.has_more" - value: false - description: "After two pages of two items the seeded set is exhausted" - - check: field_value_or_absent - path: "pagination.cursor" - allowed_values: [null] - description: "Terminal page MUST omit cursor (null also accepted for clients that explicitly clear the field)" - - check: field_value_or_absent - path: "pagination.total_count" - allowed_values: [3] - description: "total_count (when volunteered) MUST stay stable across pages and equal the seeded set size" + description: "Pagination response declares whether more sandbox accounts remain" - check: field_present path: "context" description: "Response echoes back the context object" - check: field_value path: "context.correlation_id" - value: "pagination_integrity_list_accounts--terminal_page" + value: "pagination_integrity_list_accounts--next_page" description: "Context correlation_id returned unchanged" diff --git a/dist/compliance/3.0.14/universal/storyboard-schema.yaml b/dist/compliance/3.0.14/universal/storyboard-schema.yaml index 758eabfe7d..ec0c61e97f 100644 --- a/dist/compliance/3.0.14/universal/storyboard-schema.yaml +++ b/dist/compliance/3.0.14/universal/storyboard-schema.yaml @@ -139,6 +139,13 @@ # pricing_options: # - pricing_option_id: "test-pricing" # pricing_model: "cpm" +# accounts: +# - account_id: "acct_test" +# fixture: +# brand: { domain: "acme.example" } +# operator: "buyer.example" +# billing: "operator" +# sandbox: true # creatives: # - creative_id: "campaign_hero_video" # status: "approved" @@ -149,11 +156,12 @@ # media_buys: # - media_buy_id: "mb_acme_q2_2026_auction" # status: "active" -# Each top-level key maps to a seed_* scenario (products → seed_product, -# pricing_options → seed_pricing_option, creatives → seed_creative, -# plans → seed_plan, media_buys → seed_media_buy). Foreign-key dependency DAG -# the runner MUST honor when auto-seeding: +# Each top-level key maps to a seed_* scenario (accounts → seed_account, +# products → seed_product, pricing_options → seed_pricing_option, +# creatives → seed_creative, plans → seed_plan, media_buys → seed_media_buy). +# Foreign-key dependency DAG the runner MUST honor when auto-seeding: # +# account ───→ product # product ──┬─→ pricing_option # ├─→ plan # └─→ media_buy diff --git a/dist/compliance/3.1.0-rc.6/domains/media-buy/state-machine.yaml b/dist/compliance/3.1.0-rc.6/domains/media-buy/state-machine.yaml index aa7025517a..ce0c9d4093 100644 --- a/dist/compliance/3.1.0-rc.6/domains/media-buy/state-machine.yaml +++ b/dist/compliance/3.1.0-rc.6/domains/media-buy/state-machine.yaml @@ -157,12 +157,12 @@ phases: operator: "pinnacle-agency.example" brand: domain: "acmeoutdoor.example" - start_time: "2026-07-01T00:00:00Z" + start_time: "asap" end_time: "2026-09-30T23:59:59Z" packages: - - product_id: "test-product" + - product_id: "$context.product_id" budget: 10000 - pricing_option_id: "test-pricing" + pricing_option_id: "$context.pricing_option_id" idempotency_key: "$generate:uuid_v4#media_buy_state_machine_setup_create_buy" context: diff --git a/dist/compliance/3.1.0-rc.6/protocols/media-buy/state-machine.yaml b/dist/compliance/3.1.0-rc.6/protocols/media-buy/state-machine.yaml index aa7025517a..ce0c9d4093 100644 --- a/dist/compliance/3.1.0-rc.6/protocols/media-buy/state-machine.yaml +++ b/dist/compliance/3.1.0-rc.6/protocols/media-buy/state-machine.yaml @@ -157,12 +157,12 @@ phases: operator: "pinnacle-agency.example" brand: domain: "acmeoutdoor.example" - start_time: "2026-07-01T00:00:00Z" + start_time: "asap" end_time: "2026-09-30T23:59:59Z" packages: - - product_id: "test-product" + - product_id: "$context.product_id" budget: 10000 - pricing_option_id: "test-pricing" + pricing_option_id: "$context.pricing_option_id" idempotency_key: "$generate:uuid_v4#media_buy_state_machine_setup_create_buy" context: diff --git a/dist/compliance/3.1.0-rc.6/universal/pagination-integrity-list-accounts.yaml b/dist/compliance/3.1.0-rc.6/universal/pagination-integrity-list-accounts.yaml index e542a836f7..66701f1a71 100644 --- a/dist/compliance/3.1.0-rc.6/universal/pagination-integrity-list-accounts.yaml +++ b/dist/compliance/3.1.0-rc.6/universal/pagination-integrity-list-accounts.yaml @@ -2,7 +2,7 @@ id: pagination_integrity_list_accounts version: "1.0.0" title: "Pagination continuation integrity — list_accounts" category: schema_validation -summary: "Validates the has_more=true continuation side of list_accounts pagination by creating sandbox accounts, requesting a small first page, and following its cursor once." +summary: "Validates list_accounts pagination by seeding sandbox accounts, requesting a small first page, and following its cursor once." track: core required_tools: - list_accounts @@ -20,16 +20,13 @@ narrative: | continuation side on `list_accounts`: when `has_more=true`, `cursor` must be present and usable. - The runner bootstraps three sandbox accounts via `sync_accounts`, then - lists sandbox accounts back with `max_results=2`. The first page MUST cap - at two accounts and signal continuation with `has_more=true` plus a cursor. - The runner then follows that cursor to prove the continuation token is usable. - Agents may have pre-existing sandbox accounts, so this storyboard deliberately - does not assert that the second observed page is terminal or that - `total_count` equals exactly three. The terminal side of the invariant is - covered by pagination storyboards with isolated seeded result sets; adding - equivalent account seeding with isolated lookup semantics is tracked - separately. + The runner seeds three sandbox accounts via `comply_test_controller`, then + lists sandbox accounts back with `max_results=2`. The first page MUST cap at + two accounts and signal continuation with `has_more=true` plus a cursor. The + runner then follows that cursor to prove the continuation token is usable. + The seed_account contract is session-scoped: seeded accounts are visible to + the requesting compliance principal for this run and MUST NOT rely on + production tenant data already present in the seller's sandbox. agent: interaction_model: stateful_preloaded @@ -43,18 +40,38 @@ caller: prerequisites: description: | - The storyboard bootstraps three accounts in the setup phase via - `sync_accounts`. The seed-DAG documented at - `static/compliance/source/universal/storyboard-schema.yaml` does not - include accounts (today's seed kinds are products, pricing_options, - creatives, plans, media_buys), so this storyboard deliberately - departs from the `controller_seeding: true` precedent used by - `pagination-integrity.yaml` (list_creatives) and bootstraps via - real protocol calls. Adding `seed_account` to the controller is - tracked separately; until then the create-then-list pattern keeps - the storyboard self-contained without extending the seed taxonomy. + The runner seeds three accounts through comply_test_controller.seed_account + so the pagination walk uses compliance-session state rather than assuming + pre-existing tenant fixtures. test_kit: "test-kits/acme-outdoor.yaml" - controller_seeding: false + controller_seeding: true + +fixtures: + accounts: + - account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" phases: - id: capability_discovery @@ -95,54 +112,106 @@ phases: value: "pagination_integrity_list_accounts--get_capabilities" description: "Context correlation_id returned unchanged" - - id: setup - title: "Bootstrap three test accounts" + - id: seed_accounts + title: "Seed three test accounts" narrative: | - Sync three sandbox accounts so the pagination walk has at least three - sandbox items to page through with max_results=2. + Seed three sandbox accounts so the pagination walk has at least three + sandbox items to page through with max_results=2. seed_account is + idempotent within a run: the same account_id plus same fixture succeeds, + while the same account_id with a different fixture is invalid state. steps: - - id: sync_three_accounts - title: "Sync three sandbox accounts" - narrative: | - Create three sandbox accounts — Acme c/o Pinnacle, Nova c/o Pinnacle, - and Pinnacle Direct. The agent MUST accept all three and respond with - account_id values that subsequent list_accounts calls surface. - task: sync_accounts - schema_ref: "account/sync-accounts-request.json" - response_schema_ref: "account/sync-accounts-response.json" - doc_ref: "/accounts/task-reference/sync_accounts" - comply_scenario: pagination_integrity_list_accounts + - id: seed_account_1 + title: "Seed first sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account stateful: true expected: | - Accept all three accounts and return account_id for each. - + Return success: true. sample_request: - accounts: - - brand: - domain: acme-corp.com - operator: pinnacle-media.com - billing: operator + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_1" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true - - brand: - domain: nova-brands.com - operator: pinnacle-media.com - billing: operator - sandbox: true - - brand: - domain: pinnacle-media.com - operator: pinnacle-media.com - billing: operator + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_1" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" + + - id: seed_account_2 + title: "Seed second sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_2" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_2" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - idempotency_key: "$generate:uuid_v4#pagination_integrity_list_accounts_setup_sync_three_accounts" + - id: seed_account_3 + title: "Seed third sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_3" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" context: - correlation_id: "pagination_integrity_list_accounts--setup" + correlation_id: "pagination_integrity_list_accounts--seed_3" validations: - - check: response_schema - description: "Response matches sync-accounts-response.json schema" - - check: field_present - path: "accounts" - description: "Response contains accounts array" + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - id: pagination_walk title: "Walk pages with a small max_results" @@ -155,8 +224,8 @@ phases: - id: first_page title: "Request the first page with max_results=2" narrative: | - The buyer asks for up to two sandbox accounts. The agent has at least - three matching accounts, so the request is non-terminal: callers expect + The buyer asks for up to two sandbox accounts. The compliance session + has at least three seeded accounts, so the request is non-terminal: callers expect `has_more=true` and a `cursor` they can follow to retrieve the remaining account. diff --git a/dist/compliance/3.1.0-rc.6/universal/storyboard-schema.yaml b/dist/compliance/3.1.0-rc.6/universal/storyboard-schema.yaml index 045085b4ab..4bacc1d76d 100644 --- a/dist/compliance/3.1.0-rc.6/universal/storyboard-schema.yaml +++ b/dist/compliance/3.1.0-rc.6/universal/storyboard-schema.yaml @@ -344,6 +344,13 @@ # pricing_options: # - pricing_option_id: "test-pricing" # pricing_model: "cpm" +# accounts: +# - account_id: "acct_test" +# fixture: +# brand: { domain: "acme.example" } +# operator: "buyer.example" +# billing: "operator" +# sandbox: true # creatives: # - creative_id: "campaign_hero_video" # status: "approved" @@ -354,15 +361,17 @@ # media_buys: # - media_buy_id: "mb_acme_q2_2026_auction" # status: "active" -# Each top-level key maps to a seed_* scenario (products → seed_product, -# pricing_options → seed_pricing_option, creatives → seed_creative, -# plans → seed_plan, media_buys → seed_media_buy). Storyboards that need an +# Each top-level key maps to a seed_* scenario (accounts → seed_account, +# products → seed_product, pricing_options → seed_pricing_option, +# creatives → seed_creative, plans → seed_plan, media_buys → seed_media_buy). +# Storyboards that need an # external measurement vendor catalog SHOULD add an explicit # comply_test_controller step with scenario: seed_measurement_catalog. Product # fixtures may carry `measurement_catalogs[]` entries only as a compatibility # fallback when a storyboard needs to keep that snapshot beside the product # contract. Foreign-key dependency DAG the runner MUST honor when auto-seeding: # +# account ───→ product # product ──┬─→ pricing_option # ├─→ plan # └─→ media_buy diff --git a/dist/schemas/3.1.0-rc.6/compliance/comply-test-controller-request.json b/dist/schemas/3.1.0-rc.6/compliance/comply-test-controller-request.json index 9267b25ab6..445d7c3833 100644 --- a/dist/schemas/3.1.0-rc.6/compliance/comply-test-controller-request.json +++ b/dist/schemas/3.1.0-rc.6/compliance/comply-test-controller-request.json @@ -1,12 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "/schemas/3.1.0-rc.6/compliance/comply-test-controller-request.json", + "$id": "/schemas/compliance/comply-test-controller-request.json", "title": "Comply Test Controller Request", "description": "Request payload for the comply_test_controller tool. Triggers seller-side state transitions for compliance testing. Sandbox only — sellers MUST NOT expose this tool in production. Naturally idempotent: each known `scenario` is either a lookup (`list_scenarios`) or a state-forcing operation whose target state is carried in the payload (`force_*_status`, `simulate_*`), so replays converge to the same observable state without needing an idempotency_key. The compliance harness drives this tool deterministically and does not rely on the seller's at-most-once replay cache.", "type": "object", "allOf": [ { - "$ref": "/schemas/3.1.0-rc.6/core/version-envelope.json" + "$ref": "/schemas/core/version-envelope.json" }, { "if": { @@ -28,7 +28,7 @@ ], "properties": { "status": { - "$ref": "/schemas/3.1.0-rc.6/enums/creative-status.json" + "$ref": "/schemas/enums/creative-status.json" } } } @@ -76,7 +76,7 @@ ], "properties": { "status": { - "$ref": "/schemas/3.1.0-rc.6/enums/account-status.json" + "$ref": "/schemas/enums/account-status.json" } } } @@ -103,7 +103,7 @@ ], "properties": { "status": { - "$ref": "/schemas/3.1.0-rc.6/enums/media-buy-status.json" + "$ref": "/schemas/enums/media-buy-status.json" } } } @@ -360,6 +360,27 @@ } } }, + { + "if": { + "properties": { + "scenario": { + "const": "seed_account" + } + } + }, + "then": { + "required": [ + "params" + ], + "properties": { + "params": { + "required": [ + "account_id" + ] + } + } + } + }, { "if": { "properties": { @@ -503,7 +524,7 @@ "properties": { "scenario": { "type": "string", - "description": "Test scenario to execute. 'list_scenarios' discovers supported scenarios. 'force_*' and 'simulate_*' trigger state transitions. 'force_creative_purge' destroys or tombstones a sandbox creative so account-level `creative.purged` webhooks can be observed where the seller supports the lifecycle surface. 'seed_*' scenarios pre-populate fixtures (product, pricing option, creative, plan, media buy, creative format, measurement catalog) so storyboards can reference fixture IDs and external-catalog facts without implementers guessing which fixtures the conformance suite expects. 'query_upstream_traffic' returns outbound HTTP calls the agent has made since session start (or since a caller-supplied timestamp), so storyboard runners can assert upstream side-effects via `check: upstream_traffic`. 'query_provenance_audit_observations' returns sandbox-only audit observations recorded for a submitted creative so storyboards can assert non-blocking governance observations without exposing an internal audit log on public seller responses. 'force_upstream_unavailable' marks a named upstream dependency as unreachable for the duration of the compliance session (or until the seller resets it), so storyboards can exercise stale-cache fallback paths — see the `stale_response_advisory` universal storyboard. The contract raises the bar against unintentional façades — adapters that satisfy AdCP schema requirements with synthetic placeholders. It is NOT an adversarial integrity check: adopters self-report their own traffic. Adopters MUST scope the response to traffic caused by the requesting principal's session/auth context — cross-caller traffic MUST NOT be returned, regardless of the supplied since_timestamp. Multi-tenant sandboxes MUST key the recording buffer on the comply_test_controller invocation's auth principal. Runners and sellers MUST accept unknown scenario strings — new scenarios may be added in additive releases." + "description": "Test scenario to execute. 'list_scenarios' discovers supported scenarios. 'force_*' and 'simulate_*' trigger state transitions. 'force_creative_purge' destroys or tombstones a sandbox creative so account-level `creative.purged` webhooks can be observed where the seller supports the lifecycle surface. 'seed_*' scenarios pre-populate fixtures (account, product, pricing option, creative, plan, media buy, creative format, measurement catalog) so storyboards can reference fixture IDs and external-catalog facts without implementers guessing which fixtures the conformance suite expects. 'query_upstream_traffic' returns outbound HTTP calls the agent has made since session start (or since a caller-supplied timestamp), so storyboard runners can assert upstream side-effects via `check: upstream_traffic`. 'query_provenance_audit_observations' returns sandbox-only audit observations recorded for a submitted creative so storyboards can assert non-blocking governance observations without exposing an internal audit log on public seller responses. 'force_upstream_unavailable' marks a named upstream dependency as unreachable for the duration of the compliance session (or until the seller resets it), so storyboards can exercise stale-cache fallback paths — see the `stale_response_advisory` universal storyboard. The contract raises the bar against unintentional facades — adapters that satisfy AdCP schema requirements with synthetic placeholders. It is NOT an adversarial integrity check: adopters self-report their own traffic. Adopters MUST scope the response to traffic caused by the requesting principal's session/auth context — cross-caller traffic MUST NOT be returned, regardless of the supplied since_timestamp. Multi-tenant sandboxes MUST key the recording buffer on the comply_test_controller invocation's auth principal. Runners and sellers MUST accept unknown scenario strings — new scenarios may be added in additive releases." }, "params": { "type": "object", @@ -515,7 +536,7 @@ }, "account_id": { "type": "string", - "description": "Account to transition. Used by force_account_status and simulate_budget_spend." + "description": "Account to transition, simulate, or seed. Used by force_account_status, simulate_budget_spend, and seed_account." }, "media_buy_id": { "type": "string", @@ -539,7 +560,7 @@ }, "fixture": { "type": "object", - "description": "Arbitrary fixture payload carried by seed_* scenarios. Shape matches the domain object the seed scenario creates (product, creative, plan, media buy, pricing option). Seller MAY reject malformed fixtures with INVALID_PARAMS. Kept permissive so storyboard authors can declare the minimum shape each test needs without the spec locking down every field.", + "description": "Arbitrary fixture payload carried by seed_* scenarios. Shape matches the domain object the seed scenario creates (account, product, creative, plan, media buy, pricing option). Seller MAY reject malformed fixtures with INVALID_PARAMS. Kept permissive so storyboard authors can declare the minimum shape each test needs without the spec locking down every field.", "additionalProperties": true }, "status": { @@ -559,7 +580,7 @@ "description": "Purge mode for force_creative_purge. soft retains a list_creatives tombstone; hard removes the creative entirely." }, "reason_code": { - "$ref": "/schemas/3.1.0-rc.6/enums/creative-event-reason-code.json", + "$ref": "/schemas/enums/creative-event-reason-code.json", "description": "Creative lifecycle reason code for force_creative_purge." }, "reason_detail": { @@ -629,7 +650,7 @@ "period": { "allOf": [ { - "$ref": "/schemas/3.1.0-rc.6/core/duration.json" + "$ref": "/schemas/core/duration.json" } ], "description": "Duration of the measurement window. REQUIRED when kind is `period` or `rolling`. Matches the `period` field in delivery-metrics.json's reach_window block." @@ -688,7 +709,7 @@ "description": "Average in-view duration per measurable impression in seconds. Reporting counterpart to the viewed_seconds optimization metric." }, "standard": { - "$ref": "/schemas/3.1.0-rc.6/enums/viewability-standard.json", + "$ref": "/schemas/enums/viewability-standard.json", "description": "Viewability measurement standard governing the threshold for viewable_rate and viewed_seconds." } }, @@ -724,7 +745,7 @@ "description": "Creative format ID to seed. Used by seed_creative_format. The seller MUST expose this format ID in list_creative_formats responses for the duration of the compliance session." }, "vendor": { - "$ref": "/schemas/3.1.0-rc.6/core/brand-ref.json", + "$ref": "/schemas/core/brand-ref.json", "description": "Measurement vendor whose catalog is being seeded. Used by seed_measurement_catalog. The seller MUST treat `metrics[]` as this vendor's `get_adcp_capabilities.measurement.metrics[]` snapshot for the duration of the compliance session." }, "metrics": { @@ -739,7 +760,7 @@ ], "properties": { "metric_id": { - "$ref": "/schemas/3.1.0-rc.6/core/vendor-metric-id.json" + "$ref": "/schemas/core/vendor-metric-id.json" } }, "additionalProperties": true @@ -754,17 +775,17 @@ "description": "Human-readable identifier for the upstream dependency to force unavailable (e.g., 'inventory-service', 'creative-agent'). Used by force_upstream_unavailable. When omitted, the seller marks its default upstream for the specified tool as unavailable. Sellers MUST include the same name in STALE_RESPONSE error.details.upstream.name on the affected response." }, "result": { - "$ref": "/schemas/3.1.0-rc.6/core/async-response-data.json", + "$ref": "/schemas/core/async-response-data.json", "description": "Completion payload to record against the task. Used by force_task_completion. Validates against the async-response-data union — for create_media_buy this is a CreateMediaBuyResponse with media_buy_id and packages. The seller MUST deliver this verbatim to the buyer's push_notification_config.url (the canonical 3.0 path for completion payload delivery), with all caller-supplied fields preserved (sellers MAY augment with seller-controlled fields like created_at or dsp_* IDs but MUST NOT overwrite caller-supplied values). A typed projection on the polling response is tracked for 3.1 (#3123). Sellers MUST emit INVALID_PARAMS if the payload does not validate against the response branch for the task's original method, and MAY reject payloads exceeding 256 KB with INVALID_PARAMS." } }, "additionalProperties": true }, "context": { - "$ref": "/schemas/3.1.0-rc.6/core/context.json" + "$ref": "/schemas/core/context.json" }, "ext": { - "$ref": "/schemas/3.1.0-rc.6/core/ext.json" + "$ref": "/schemas/core/ext.json" }, "account": { "type": "object", diff --git a/scripts/lint-storyboard-contradictions.cjs b/scripts/lint-storyboard-contradictions.cjs index 33a3bc52a1..e2100243d8 100644 --- a/scripts/lint-storyboard-contradictions.cjs +++ b/scripts/lint-storyboard-contradictions.cjs @@ -237,6 +237,7 @@ function fingerprintRequest(req) { * eliminates a false-negative envelope in the env fingerprint. */ const FIXTURE_CATEGORY_PRIMARY_ID = { + accounts: 'account_id', products: 'product_id', pricing_options: 'pricing_option_id', creatives: 'creative_id', diff --git a/server/src/training-agent/account-handlers.ts b/server/src/training-agent/account-handlers.ts index 0608dac608..178ecf04d2 100644 --- a/server/src/training-agent/account-handlers.ts +++ b/server/src/training-agent/account-handlers.ts @@ -453,6 +453,78 @@ export function resolveAccountIdForRef( return account?.accountId; } +export function seedAccountFixture( + args: ToolArgs, + ctx: TrainingContext, +): { success: boolean; message?: string; error?: string; error_detail?: string } { + const identityError = durableAccountIdentityError(ctx); + if (identityError) { + return { + success: false, + error: identityError.errors[0]?.code ?? 'AUTH_REQUIRED', + error_detail: identityError.errors[0]?.message ?? 'Durable account tools require caller identity', + }; + } + + const params = ((args as Record).params ?? {}) as Record; + const accountId = params.account_id; + if (typeof accountId !== 'string' || accountId.length === 0) { + return { success: false, error: 'INVALID_PARAMS', error_detail: 'params.account_id is required for seed_account' }; + } + + const fixture = (params.fixture ?? {}) as Record; + const brand = fixture.brand as { domain?: string; brand_id?: string; name?: string } | undefined; + const operator = fixture.operator; + if (!brand?.domain) { + return { success: false, error: 'INVALID_PARAMS', error_detail: 'params.fixture.brand.domain is required for seed_account' }; + } + if (typeof operator !== 'string' || operator.length === 0) { + return { success: false, error: 'INVALID_PARAMS', error_detail: 'params.fixture.operator is required for seed_account' }; + } + + const billing = typeof fixture.billing === 'string' ? fixture.billing : 'operator'; + const status = typeof fixture.status === 'string' ? fixture.status : 'active'; + const now = new Date().toISOString(); + const sessionKey = sessionKeyFromArgs({}, ctx.mode, ctx.userId, ctx.moduleId); + const accounts = getAccountMap(sessionKey, ctx.principal); + const key = accountKey(brand as { domain: string; brand_id?: string }, operator); + const existing = accounts.get(key) + ?? findAccountByIdAcrossSessions(accountId, ctx.principal); + + const state: AccountState = { + accountId, + brand: brand as { domain: string; brand_id?: string; name?: string }, + operator, + billing, + paymentTerms: typeof fixture.payment_terms === 'string' ? fixture.payment_terms : 'net_30', + status, + accountScope: typeof fixture.account_scope === 'string' ? fixture.account_scope : 'operator_brand', + sandbox: fixture.sandbox !== false, + rateCard: typeof fixture.rate_card === 'string' ? fixture.rate_card : 'sandbox', + creditLimit: undefined, + governanceAgents: [], + notificationConfigs: [], + notificationConfigsTouched: false, + syncedAt: now, + }; + + if (existing) { + const existingComparable = JSON.stringify(accountStateToWire(existing)); + const nextComparable = JSON.stringify(accountStateToWire(state)); + if (existingComparable !== nextComparable) { + return { + success: false, + error: 'INVALID_STATE', + error_detail: `account_id "${accountId}" was already seeded with a different fixture - seed_account is idempotent`, + }; + } + return { success: true, message: `account_id "${accountId}" already seeded with the same fixture` }; + } + + accounts.set(key, state); + return { success: true, message: `Account "${accountId}" seeded` }; +} + // Compliance fixture pool — used when the session has no synced accounts, so // storyboards that rely on stable account IDs work without prior sync_accounts. function getComplianceAccounts(): AccountWireShape[] { @@ -1064,10 +1136,12 @@ export function handleListAccounts(args: ToolArgs, ctx: TrainingContext): object const preferFixtureAccounts = ctx.storyboardCompat?.version === '3.0'; const exactAccountFilter = hasExactAccountFilter(req.account) && (req.sandbox !== true || Boolean(req.account?.account_id)); - const scopedAccounts = !preferFixtureAccounts ? accountsForPrincipal(ctx.principal) : []; + const scopedAccounts = accountsForPrincipal(ctx.principal); let accounts: AccountWireShape[] = preferFixtureAccounts - ? getComplianceAccounts() + ? scopedAccounts.length > 0 + ? scopedAccounts.map(accountStateToWire) + : getComplianceAccounts() : scopedAccounts.length > 0 ? scopedAccounts.map(accountStateToWire) : accountMap.size > 0 diff --git a/server/src/training-agent/comply-test-controller.ts b/server/src/training-agent/comply-test-controller.ts index 2abe8bd280..53f7fc9129 100644 --- a/server/src/training-agent/comply-test-controller.ts +++ b/server/src/training-agent/comply-test-controller.ts @@ -36,7 +36,7 @@ import type { import { getSession, sessionKeyFromArgs } from './state.js'; import { getAgentUrl } from './config.js'; import { randomUUID } from 'node:crypto'; -import { getAccountNotificationSubscribers } from './account-handlers.js'; +import { getAccountNotificationSubscribers, seedAccountFixture } from './account-handlers.js'; import { emitAccountNotificationWebhook } from './webhooks.js'; import { buildCatalog } from './product-factory.js'; import { getAllSignals } from './signal-providers.js'; @@ -769,6 +769,7 @@ const LOCAL_SCENARIOS = [ 'force_task_completion', 'force_creative_purge', 'force_wholesale_feed_webhook', + 'seed_account', 'seed_creative_format', 'seed_measurement_catalog', 'query_provenance_audit_observations', @@ -916,6 +917,9 @@ export async function handleComplyTestController(args: ToolArgs, ctx: TrainingCo } return handleForceWholesaleFeedWebhook(sessionKey, ctx.principal, rawArgs); } + if (scenario === 'seed_account') { + return seedAccountFixture(rawArgs as ToolArgs, ctx); + } if (scenario === 'seed_measurement_catalog') { return handleSeedMeasurementCatalog(session, rawArgs); } diff --git a/server/src/training-agent/task-handlers.ts b/server/src/training-agent/task-handlers.ts index 8879873ff5..6de7980d33 100644 --- a/server/src/training-agent/task-handlers.ts +++ b/server/src/training-agent/task-handlers.ts @@ -5824,6 +5824,7 @@ export async function handleGetAdcpCapabilities(args: ToolArgs, ctx: TrainingCon 'force_session_status', 'simulate_delivery', 'simulate_budget_spend', + 'seed_account', 'seed_product', 'seed_pricing_option', 'seed_creative', diff --git a/server/src/training-agent/tenants/router.ts b/server/src/training-agent/tenants/router.ts index 866daa84fb..c514374e1d 100644 --- a/server/src/training-agent/tenants/router.ts +++ b/server/src/training-agent/tenants/router.ts @@ -39,6 +39,7 @@ const SALES_CURRENT_SCENARIOS = [ 'force_task_completion', 'force_creative_purge', 'force_upstream_unavailable', + 'seed_account', 'seed_product', 'seed_pricing_option', 'seed_creative', diff --git a/server/src/training-agent/tenants/tenant-smoke.test.ts b/server/src/training-agent/tenants/tenant-smoke.test.ts index 411dce45ce..cd9fdb46bc 100644 --- a/server/src/training-agent/tenants/tenant-smoke.test.ts +++ b/server/src/training-agent/tenants/tenant-smoke.test.ts @@ -21,6 +21,7 @@ const SALES_CURRENT_SCENARIOS = [ 'force_create_media_buy_arm', 'force_task_completion', 'force_creative_purge', + 'seed_account', 'seed_product', 'seed_pricing_option', 'seed_creative', diff --git a/server/tests/unit/comply-test-controller.test.ts b/server/tests/unit/comply-test-controller.test.ts index f3cc245a26..25ff40f436 100644 --- a/server/tests/unit/comply-test-controller.test.ts +++ b/server/tests/unit/comply-test-controller.test.ts @@ -12,6 +12,7 @@ import { } from '../../src/training-agent/state.js'; import { MUTATING_TOOLS, clearIdempotencyCache } from '../../src/training-agent/idempotency.js'; import type { TrainingContext } from '../../src/training-agent/types.js'; +import { clearAccountStore } from '../../src/training-agent/account-handlers.js'; const DEFAULT_CTX: TrainingContext = { mode: 'open' }; const ACCOUNT = { brand: { domain: 'comply-test.example.com' }, operator: 'comply-tester', sandbox: true }; @@ -141,6 +142,7 @@ describe('comply_test_controller', () => { beforeEach(() => { clearSessions(); + clearAccountStore(); invalidateCache(); clearTaskStore(); clearIdempotencyCache(); @@ -186,6 +188,7 @@ describe('comply_test_controller', () => { 'force_task_completion', 'force_creative_purge', 'force_wholesale_feed_webhook', + 'seed_account', 'seed_creative_format', 'seed_measurement_catalog', 'query_provenance_audit_observations', @@ -193,12 +196,70 @@ describe('comply_test_controller', () => { ])); // Catch silent drift in either direction (entries removed, or new ones // not yet documented in this assertion). - expect(scenarios.length).toBe(19); + expect(scenarios.length).toBe(20); // Dedup invariant — see SCENARIO_ENUM dedup in the wrapper. expect(new Set(scenarios).size).toBe(scenarios.length); }); }); + describe('seed_account', () => { + it('seeds accounts into list_accounts for pagination checks', async () => { + for (const n of [1, 2, 3]) { + const fixture = { + brand: { domain: `seed-account-${n}.example` }, + operator: 'seed-operator.example', + billing: 'operator', + sandbox: true, + status: 'active', + }; + const { result } = await simulateCallTool(server, 'comply_test_controller', { + scenario: 'seed_account', + account: { sandbox: true, account_id: `acct_seed_account_${n}` }, + params: { + account_id: `acct_seed_account_${n}`, + fixture, + }, + }); + expect(result.success).toBe(true); + + const { result: repeatResult } = await simulateCallTool(server, 'comply_test_controller', { + scenario: 'seed_account', + account: { sandbox: true, account_id: `acct_seed_account_${n}` }, + params: { + account_id: `acct_seed_account_${n}`, + fixture, + }, + }); + expect(repeatResult.success).toBe(true); + } + + const { result } = await simulateCallTool(server, 'list_accounts', { + sandbox: true, + pagination: { max_results: 2 }, + }); + + const accounts = result.accounts as Array<{ account_id: string }>; + expect(accounts.map(a => a.account_id)).toEqual(expect.arrayContaining([ + 'acct_seed_account_1', + 'acct_seed_account_2', + ])); + expect(result.pagination).toMatchObject({ has_more: true }); + expect((result.pagination as { cursor?: string }).cursor).toBeTruthy(); + + const threeZeroServer = createTrainingAgentServer({ ...DEFAULT_CTX, storyboardCompat: { version: '3.0' } }); + const { result: threeZeroResult } = await simulateCallTool(threeZeroServer, 'list_accounts', { + sandbox: true, + pagination: { max_results: 2 }, + }); + const threeZeroAccounts = threeZeroResult.accounts as Array<{ account_id: string }>; + expect(threeZeroAccounts.map(a => a.account_id)).toEqual(expect.arrayContaining([ + 'acct_seed_account_1', + 'acct_seed_account_2', + ])); + expect(threeZeroResult.pagination).toMatchObject({ has_more: true }); + }); + }); + describe('query_provenance_audit_observations', () => { it('returns audit observations recorded during accepted provenance sync', async () => { const creativeId = 'creative-audit-directed'; diff --git a/server/tests/unit/training-agent.test.ts b/server/tests/unit/training-agent.test.ts index 5af6bfcfde..0cf296831a 100644 --- a/server/tests/unit/training-agent.test.ts +++ b/server/tests/unit/training-agent.test.ts @@ -7594,6 +7594,7 @@ describe('get_adcp_capabilities handler', () => { 'force_session_status', 'simulate_delivery', 'simulate_budget_spend', + 'seed_account', 'seed_product', 'seed_pricing_option', 'seed_creative', diff --git a/static/compliance/source/protocols/media-buy/state-machine.yaml b/static/compliance/source/protocols/media-buy/state-machine.yaml index aa7025517a..ce0c9d4093 100644 --- a/static/compliance/source/protocols/media-buy/state-machine.yaml +++ b/static/compliance/source/protocols/media-buy/state-machine.yaml @@ -157,12 +157,12 @@ phases: operator: "pinnacle-agency.example" brand: domain: "acmeoutdoor.example" - start_time: "2026-07-01T00:00:00Z" + start_time: "asap" end_time: "2026-09-30T23:59:59Z" packages: - - product_id: "test-product" + - product_id: "$context.product_id" budget: 10000 - pricing_option_id: "test-pricing" + pricing_option_id: "$context.pricing_option_id" idempotency_key: "$generate:uuid_v4#media_buy_state_machine_setup_create_buy" context: diff --git a/static/compliance/source/universal/pagination-integrity-list-accounts.yaml b/static/compliance/source/universal/pagination-integrity-list-accounts.yaml index e542a836f7..66701f1a71 100644 --- a/static/compliance/source/universal/pagination-integrity-list-accounts.yaml +++ b/static/compliance/source/universal/pagination-integrity-list-accounts.yaml @@ -2,7 +2,7 @@ id: pagination_integrity_list_accounts version: "1.0.0" title: "Pagination continuation integrity — list_accounts" category: schema_validation -summary: "Validates the has_more=true continuation side of list_accounts pagination by creating sandbox accounts, requesting a small first page, and following its cursor once." +summary: "Validates list_accounts pagination by seeding sandbox accounts, requesting a small first page, and following its cursor once." track: core required_tools: - list_accounts @@ -20,16 +20,13 @@ narrative: | continuation side on `list_accounts`: when `has_more=true`, `cursor` must be present and usable. - The runner bootstraps three sandbox accounts via `sync_accounts`, then - lists sandbox accounts back with `max_results=2`. The first page MUST cap - at two accounts and signal continuation with `has_more=true` plus a cursor. - The runner then follows that cursor to prove the continuation token is usable. - Agents may have pre-existing sandbox accounts, so this storyboard deliberately - does not assert that the second observed page is terminal or that - `total_count` equals exactly three. The terminal side of the invariant is - covered by pagination storyboards with isolated seeded result sets; adding - equivalent account seeding with isolated lookup semantics is tracked - separately. + The runner seeds three sandbox accounts via `comply_test_controller`, then + lists sandbox accounts back with `max_results=2`. The first page MUST cap at + two accounts and signal continuation with `has_more=true` plus a cursor. The + runner then follows that cursor to prove the continuation token is usable. + The seed_account contract is session-scoped: seeded accounts are visible to + the requesting compliance principal for this run and MUST NOT rely on + production tenant data already present in the seller's sandbox. agent: interaction_model: stateful_preloaded @@ -43,18 +40,38 @@ caller: prerequisites: description: | - The storyboard bootstraps three accounts in the setup phase via - `sync_accounts`. The seed-DAG documented at - `static/compliance/source/universal/storyboard-schema.yaml` does not - include accounts (today's seed kinds are products, pricing_options, - creatives, plans, media_buys), so this storyboard deliberately - departs from the `controller_seeding: true` precedent used by - `pagination-integrity.yaml` (list_creatives) and bootstraps via - real protocol calls. Adding `seed_account` to the controller is - tracked separately; until then the create-then-list pattern keeps - the storyboard self-contained without extending the seed taxonomy. + The runner seeds three accounts through comply_test_controller.seed_account + so the pagination walk uses compliance-session state rather than assuming + pre-existing tenant fixtures. test_kit: "test-kits/acme-outdoor.yaml" - controller_seeding: false + controller_seeding: true + +fixtures: + accounts: + - account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" + - account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" phases: - id: capability_discovery @@ -95,54 +112,106 @@ phases: value: "pagination_integrity_list_accounts--get_capabilities" description: "Context correlation_id returned unchanged" - - id: setup - title: "Bootstrap three test accounts" + - id: seed_accounts + title: "Seed three test accounts" narrative: | - Sync three sandbox accounts so the pagination walk has at least three - sandbox items to page through with max_results=2. + Seed three sandbox accounts so the pagination walk has at least three + sandbox items to page through with max_results=2. seed_account is + idempotent within a run: the same account_id plus same fixture succeeds, + while the same account_id with a different fixture is invalid state. steps: - - id: sync_three_accounts - title: "Sync three sandbox accounts" - narrative: | - Create three sandbox accounts — Acme c/o Pinnacle, Nova c/o Pinnacle, - and Pinnacle Direct. The agent MUST accept all three and respond with - account_id values that subsequent list_accounts calls surface. - task: sync_accounts - schema_ref: "account/sync-accounts-request.json" - response_schema_ref: "account/sync-accounts-response.json" - doc_ref: "/accounts/task-reference/sync_accounts" - comply_scenario: pagination_integrity_list_accounts + - id: seed_account_1 + title: "Seed first sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account stateful: true expected: | - Accept all three accounts and return account_id for each. - + Return success: true. sample_request: - accounts: - - brand: - domain: acme-corp.com - operator: pinnacle-media.com - billing: operator + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_1" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_1" + fixture: + brand: + domain: "pagination-acme-1.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true - - brand: - domain: nova-brands.com - operator: pinnacle-media.com - billing: operator - sandbox: true - - brand: - domain: pinnacle-media.com - operator: pinnacle-media.com - billing: operator + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_1" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" + + - id: seed_account_2 + title: "Seed second sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_2" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_2" + fixture: + brand: + domain: "pagination-acme-2.example" + operator: "pagination-pinnacle.example" + billing: "operator" sandbox: true + status: "active" + context: + correlation_id: "pagination_integrity_list_accounts--seed_2" + validations: + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - idempotency_key: "$generate:uuid_v4#pagination_integrity_list_accounts_setup_sync_three_accounts" + - id: seed_account_3 + title: "Seed third sandbox account" + task: comply_test_controller + schema_ref: "compliance/comply-test-controller-request.json" + response_schema_ref: "compliance/comply-test-controller-response.json" + comply_scenario: seed_account + stateful: true + expected: | + Return success: true. + sample_request: + account: + sandbox: true + account_id: "acct_pagination_integrity_accounts_3" + scenario: "seed_account" + params: + account_id: "acct_pagination_integrity_accounts_3" + fixture: + brand: + domain: "pagination-acme-3.example" + operator: "pagination-pinnacle.example" + billing: "operator" + sandbox: true + status: "active" context: - correlation_id: "pagination_integrity_list_accounts--setup" + correlation_id: "pagination_integrity_list_accounts--seed_3" validations: - - check: response_schema - description: "Response matches sync-accounts-response.json schema" - - check: field_present - path: "accounts" - description: "Response contains accounts array" + - check: field_value + path: "success" + value: true + description: "Seed acknowledged" - id: pagination_walk title: "Walk pages with a small max_results" @@ -155,8 +224,8 @@ phases: - id: first_page title: "Request the first page with max_results=2" narrative: | - The buyer asks for up to two sandbox accounts. The agent has at least - three matching accounts, so the request is non-terminal: callers expect + The buyer asks for up to two sandbox accounts. The compliance session + has at least three seeded accounts, so the request is non-terminal: callers expect `has_more=true` and a `cursor` they can follow to retrieve the remaining account. diff --git a/static/compliance/source/universal/storyboard-schema.yaml b/static/compliance/source/universal/storyboard-schema.yaml index 045085b4ab..4bacc1d76d 100644 --- a/static/compliance/source/universal/storyboard-schema.yaml +++ b/static/compliance/source/universal/storyboard-schema.yaml @@ -344,6 +344,13 @@ # pricing_options: # - pricing_option_id: "test-pricing" # pricing_model: "cpm" +# accounts: +# - account_id: "acct_test" +# fixture: +# brand: { domain: "acme.example" } +# operator: "buyer.example" +# billing: "operator" +# sandbox: true # creatives: # - creative_id: "campaign_hero_video" # status: "approved" @@ -354,15 +361,17 @@ # media_buys: # - media_buy_id: "mb_acme_q2_2026_auction" # status: "active" -# Each top-level key maps to a seed_* scenario (products → seed_product, -# pricing_options → seed_pricing_option, creatives → seed_creative, -# plans → seed_plan, media_buys → seed_media_buy). Storyboards that need an +# Each top-level key maps to a seed_* scenario (accounts → seed_account, +# products → seed_product, pricing_options → seed_pricing_option, +# creatives → seed_creative, plans → seed_plan, media_buys → seed_media_buy). +# Storyboards that need an # external measurement vendor catalog SHOULD add an explicit # comply_test_controller step with scenario: seed_measurement_catalog. Product # fixtures may carry `measurement_catalogs[]` entries only as a compatibility # fallback when a storyboard needs to keep that snapshot beside the product # contract. Foreign-key dependency DAG the runner MUST honor when auto-seeding: # +# account ───→ product # product ──┬─→ pricing_option # ├─→ plan # └─→ media_buy diff --git a/static/schemas/source/compliance/comply-test-controller-request.json b/static/schemas/source/compliance/comply-test-controller-request.json index 4b8db96a27..445d7c3833 100644 --- a/static/schemas/source/compliance/comply-test-controller-request.json +++ b/static/schemas/source/compliance/comply-test-controller-request.json @@ -360,6 +360,27 @@ } } }, + { + "if": { + "properties": { + "scenario": { + "const": "seed_account" + } + } + }, + "then": { + "required": [ + "params" + ], + "properties": { + "params": { + "required": [ + "account_id" + ] + } + } + } + }, { "if": { "properties": { @@ -503,7 +524,7 @@ "properties": { "scenario": { "type": "string", - "description": "Test scenario to execute. 'list_scenarios' discovers supported scenarios. 'force_*' and 'simulate_*' trigger state transitions. 'force_creative_purge' destroys or tombstones a sandbox creative so account-level `creative.purged` webhooks can be observed where the seller supports the lifecycle surface. 'seed_*' scenarios pre-populate fixtures (product, pricing option, creative, plan, media buy, creative format, measurement catalog) so storyboards can reference fixture IDs and external-catalog facts without implementers guessing which fixtures the conformance suite expects. 'query_upstream_traffic' returns outbound HTTP calls the agent has made since session start (or since a caller-supplied timestamp), so storyboard runners can assert upstream side-effects via `check: upstream_traffic`. 'query_provenance_audit_observations' returns sandbox-only audit observations recorded for a submitted creative so storyboards can assert non-blocking governance observations without exposing an internal audit log on public seller responses. 'force_upstream_unavailable' marks a named upstream dependency as unreachable for the duration of the compliance session (or until the seller resets it), so storyboards can exercise stale-cache fallback paths — see the `stale_response_advisory` universal storyboard. The contract raises the bar against unintentional façades — adapters that satisfy AdCP schema requirements with synthetic placeholders. It is NOT an adversarial integrity check: adopters self-report their own traffic. Adopters MUST scope the response to traffic caused by the requesting principal's session/auth context — cross-caller traffic MUST NOT be returned, regardless of the supplied since_timestamp. Multi-tenant sandboxes MUST key the recording buffer on the comply_test_controller invocation's auth principal. Runners and sellers MUST accept unknown scenario strings — new scenarios may be added in additive releases." + "description": "Test scenario to execute. 'list_scenarios' discovers supported scenarios. 'force_*' and 'simulate_*' trigger state transitions. 'force_creative_purge' destroys or tombstones a sandbox creative so account-level `creative.purged` webhooks can be observed where the seller supports the lifecycle surface. 'seed_*' scenarios pre-populate fixtures (account, product, pricing option, creative, plan, media buy, creative format, measurement catalog) so storyboards can reference fixture IDs and external-catalog facts without implementers guessing which fixtures the conformance suite expects. 'query_upstream_traffic' returns outbound HTTP calls the agent has made since session start (or since a caller-supplied timestamp), so storyboard runners can assert upstream side-effects via `check: upstream_traffic`. 'query_provenance_audit_observations' returns sandbox-only audit observations recorded for a submitted creative so storyboards can assert non-blocking governance observations without exposing an internal audit log on public seller responses. 'force_upstream_unavailable' marks a named upstream dependency as unreachable for the duration of the compliance session (or until the seller resets it), so storyboards can exercise stale-cache fallback paths — see the `stale_response_advisory` universal storyboard. The contract raises the bar against unintentional facades — adapters that satisfy AdCP schema requirements with synthetic placeholders. It is NOT an adversarial integrity check: adopters self-report their own traffic. Adopters MUST scope the response to traffic caused by the requesting principal's session/auth context — cross-caller traffic MUST NOT be returned, regardless of the supplied since_timestamp. Multi-tenant sandboxes MUST key the recording buffer on the comply_test_controller invocation's auth principal. Runners and sellers MUST accept unknown scenario strings — new scenarios may be added in additive releases." }, "params": { "type": "object", @@ -515,7 +536,7 @@ }, "account_id": { "type": "string", - "description": "Account to transition. Used by force_account_status and simulate_budget_spend." + "description": "Account to transition, simulate, or seed. Used by force_account_status, simulate_budget_spend, and seed_account." }, "media_buy_id": { "type": "string", @@ -539,7 +560,7 @@ }, "fixture": { "type": "object", - "description": "Arbitrary fixture payload carried by seed_* scenarios. Shape matches the domain object the seed scenario creates (product, creative, plan, media buy, pricing option). Seller MAY reject malformed fixtures with INVALID_PARAMS. Kept permissive so storyboard authors can declare the minimum shape each test needs without the spec locking down every field.", + "description": "Arbitrary fixture payload carried by seed_* scenarios. Shape matches the domain object the seed scenario creates (account, product, creative, plan, media buy, pricing option). Seller MAY reject malformed fixtures with INVALID_PARAMS. Kept permissive so storyboard authors can declare the minimum shape each test needs without the spec locking down every field.", "additionalProperties": true }, "status": { From 4f1f0c8e826378c66beb208ce03375789bcae7a2 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 3 Jun 2026 11:58:45 -0400 Subject: [PATCH 2/2] Add empty changeset --- .changeset/wild-rocks-yell.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changeset/wild-rocks-yell.md diff --git a/.changeset/wild-rocks-yell.md b/.changeset/wild-rocks-yell.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/wild-rocks-yell.md @@ -0,0 +1,2 @@ +--- +---