From be510263c1c52f84e6b414ec3c8eafb4f16611f1 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 11 May 2026 10:07:25 -0400 Subject: [PATCH 1/3] spec(idempotency): add rules 9 (concurrent retries) and 10 (downstream reconciliation); introduce IDEMPOTENCY_IN_FLIGHT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new normative rules in L1/security.mdx#idempotency close the last gaps the existing 1-8 left underspecified — what happens when a second request arrives while the first is still running, and what a seller must do about downstream calls whose outcome it lost. Rule 9 (first-insert-wins). Sellers MUST resolve concurrent same-key retries deterministically via INSERT ... ON CONFLICT DO NOTHING on (authenticated_agent, account_id, idempotency_key), and MAY pick one of two policies: wait-and-replay (block, return cached) or reject-and-redirect (return new IDEMPOTENCY_IN_FLIGHT with error.details.retry_after). Verified against the Python sales-agent (Wonderstruck) — its wait-and-replay implementation passes out of the box; the new rule documents existing canonical behavior. Rule 10 (downstream reconciliation). When a seller invokes a downstream system during request handling, it MUST adopt either write-claim- before-invoke or thread-buyer-key. Best-effort dedup on downstream response inspection is explicitly forbidden. New error code IDEMPOTENCY_IN_FLIGHT (held for 3.1 per wire-stability policy). Recovery: transient. Buyers MUST retry with the SAME key after retry_after — minting a fresh key here turns a safe retry into a double-execution race. Storyboard concurrent_retry phase added to static/compliance/source/universal/idempotency.yaml, with two new cross-response check kinds (cross_response_count_distinct, cross_response_field_equal). Runner contract documented in static/compliance/source/test-kits/parallel-dispatch-runner.yaml. Runners without parallel-dispatch support skip with not_applicable. Author skill and calling-an-agent.mdx updated so buyers retry rather than mint fresh keys on the new error code. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/idempotency-rule-9-and-10.md | 17 ++ docs/building/by-layer/L1/security.mdx | 20 ++ docs/protocol/calling-an-agent.mdx | 3 +- scripts/error-code-drift-dispositions.json | 5 + skills/call-adcp-agent/SKILL.md | 3 +- .../test-kits/parallel-dispatch-runner.yaml | 196 ++++++++++++++++++ .../source/universal/idempotency.yaml | 121 ++++++++++- .../universal/runner-output-contract.yaml | 2 + static/schemas/source/enums/error-code.json | 6 + 9 files changed, 369 insertions(+), 4 deletions(-) create mode 100644 .changeset/idempotency-rule-9-and-10.md create mode 100644 static/compliance/source/test-kits/parallel-dispatch-runner.yaml diff --git a/.changeset/idempotency-rule-9-and-10.md b/.changeset/idempotency-rule-9-and-10.md new file mode 100644 index 0000000000..b9988f754b --- /dev/null +++ b/.changeset/idempotency-rule-9-and-10.md @@ -0,0 +1,17 @@ +--- +"adcontextprotocol": minor +--- + +spec(idempotency): add normative rules for concurrent retries and downstream reconciliation; introduce `IDEMPOTENCY_IN_FLIGHT` + +Two new normative rules in `L1/security.mdx#idempotency`: + +**Rule 9 — Concurrent retries / first-insert-wins.** A second request carrying the same `(authenticated_agent, account_id, idempotency_key)` MAY arrive while the first is still executing. Sellers MUST resolve the race deterministically (`INSERT … ON CONFLICT DO NOTHING` on the scope tuple) and MAY pick one of two policies, behaving consistently: **wait-and-replay** (block the second request until the first completes, return cached response with `replayed: true`), or **reject-and-redirect** (return new `IDEMPOTENCY_IN_FLIGHT` code with `error.details.retry_after`). Same key with a *different* canonical payload during the in-flight window still returns `IDEMPOTENCY_CONFLICT` (rule 5). Verified against the canonical Python sales-agent (Wonderstruck) — its wait-and-replay implementation passes the new rule out of the box. + +**Rule 10 — Crossing service boundaries / downstream reconciliation.** When a seller invokes a downstream system (SSP, ad server, payment provider) during request handling, "errors don't cache" (rule 3) is necessary but not sufficient — a crash between downstream-accepts and local-persist leaves the seller in a "downstream unknown" state. Sellers MUST adopt one of two patterns for every downstream call whose duplicate-invocation has business consequences: **write-claim-before-invoke** (persist a claim row with `downstream_request_id` before invoking; reconcile on retry by querying the downstream by that id) or **thread-buyer-key** (pass the buyer's `idempotency_key` or a deterministic seller-side derivative as the downstream's own idempotency key). The pattern "best-effort dedup on downstream response inspection" is explicitly forbidden. + +**New error code: `IDEMPOTENCY_IN_FLIGHT`** (held for 3.1 per the wire-stability policy). Recovery: transient. Buyers MUST retry with the **same** `idempotency_key` after `error.details.retry_after` — minting a fresh key on this code turns a safe retry into a double-execution race. + +**Storyboard coverage.** `static/compliance/source/universal/idempotency.yaml` gains a `concurrent_retry` phase using two new cross-response check kinds (`cross_response_count_distinct`, `cross_response_field_equal`) that operate on the resolved response set across N parallel dispatches. The runner contract is documented in the new `test-kits/parallel-dispatch-runner.yaml`; runners without parallel-dispatch support skip the phase with a stable not_applicable marker. SDK/runner implementation tracked separately (adcp-client follow-up). + +Author skill (`skills/call-adcp-agent/SKILL.md`) and the buyer-facing `docs/protocol/calling-an-agent.mdx` updated so buyers know to wait-and-retry on `IDEMPOTENCY_IN_FLIGHT` rather than mint a fresh key. diff --git a/docs/building/by-layer/L1/security.mdx b/docs/building/by-layer/L1/security.mdx index 2afa5db1d4..dc1cffee23 100644 --- a/docs/building/by-layer/L1/security.mdx +++ b/docs/building/by-layer/L1/security.mdx @@ -299,6 +299,26 @@ This section applies only to AdCP task requests. OpenRTB bid streams have their The ceiling is per `(authenticated_agent, account)` — the same scope as the idempotency key itself (bullet 1) — so a multi-account agency does not have its per-account budgets collapsed into a single shared quota. `RATE_LIMITED` rejections MUST populate `retry_after` (seconds) per the [error handling taxonomy](/docs/building/by-layer/L3/error-handling#rate-limit-handling) and MUST NOT be cached as idempotency responses (rule 3: only successful responses are cached). Sellers SHOULD enforce `retry_after` as a cheap rejection floor — a buyer retrying before `retry_after` elapses SHOULD hit a pre-auth token bucket (e.g., at a reverse-proxy layer) rather than re-entering the full schema-validate-and-cache-check pipeline on every retry. Without this discipline, misbehaving buyers can amplify load on the rate-limiter itself. +9. **Concurrent retries — first-insert-wins.** A second request carrying the same `(authenticated_agent, account_id, idempotency_key)` MAY arrive while the first request is still executing — most commonly when the buyer's transport timeout fires before the seller's downstream call returns, and the buyer retries. Sellers MUST resolve the race deterministically; they MUST NOT execute the side effect twice and MUST NOT silently drop the second request. Resolution is a `(unique constraint, INSERT … ON CONFLICT DO NOTHING)` pattern on the scope tuple: the first row to land owns execution; subsequent requests observe an existing row whose response slot is not yet populated. + + Sellers MUST handle the second request by one of two policies and MUST behave consistently across calls — clients infer the policy from the first response within a session and apply it to subsequent retries: + + - **Wait-and-replay** (preferred for fast operations, <5s typical): the seller blocks the second request until the first completes, then returns the cached response with `replayed: true`. Total wall-time for the second call is bounded by the seller's request-timeout budget. + - **Reject-and-redirect** (preferred for slow operations involving long-running downstream calls): the seller returns `IDEMPOTENCY_IN_FLIGHT` immediately, with `error.details.retry_after` (seconds, integer) populated based on the first request's elapsed time and expected completion. Buyers MUST retry with the same `idempotency_key` after the hint elapses — a buyer that mints a fresh key on `IDEMPOTENCY_IN_FLIGHT` turns a safe retry into the exact double-execution race this rule prevents. + + A second request with the same key AND a *different* canonical payload during the in-flight window MUST return `IDEMPOTENCY_CONFLICT` (rule 5), not `IDEMPOTENCY_IN_FLIGHT` — the canonical-form mismatch is computable at INSERT time against the row's stored hash, so the conflict is detectable without waiting for the first request's response. Per rule 3, if the first request ultimately fails (validation error, downstream timeout, internal error), the `(in_flight)` row is released — the key returns to "never seen" state and a subsequent retry re-executes from scratch. Sellers MUST NOT leak the in-flight state across the scope boundary: an attacker probing a candidate key MUST receive the same response shape and timing whether the row exists, is in flight, or has never existed. + +10. **Crossing service boundaries — downstream reconciliation.** Sellers commonly invoke downstream systems during request handling — SSP/ad-server calls on `create_media_buy`, payment-provider calls on billing operations, governance-agent calls on `check_governance`. These calls have their own failure modes that can leave the seller in a "downstream unknown" state: the network connection dropped after the downstream accepted the request but before its response arrived; the seller process crashed mid-call; a region failover swapped the worker before the response was persisted. Rule 3 (only successful responses cached) is necessary but not sufficient: a seller that simply doesn't cache and re-executes on retry will double-invoke the downstream and create duplicate side effects there. + + Sellers MUST adopt one of two reconciliation patterns for every downstream call whose duplicate-invocation has business consequences (resource creation, payment movement, irreversible state change). Read-only downstream calls are exempt. + + - **Write-claim-before-invoke (preferred default).** Before invoking the downstream, the seller persists a "claim" row in the same transaction as the idempotency cache row — typically `{idempotency_key, downstream_provider, downstream_request_id, status: 'invoked', invoked_at}` — using the seller-generated `downstream_request_id` it will pass to the downstream as the downstream's own correlation/idempotency identifier. On retry, before invoking the downstream again, the seller MUST look up the claim row by `(idempotency_key, downstream_provider)` and reconcile: query the downstream by `downstream_request_id` to determine the true outcome, then resume cache population from there. The seller MUST NOT treat a missing local record as "downstream call did not happen" — a crash between downstream-accepts and local-persist is exactly the case where it did happen and the local record is missing. + - **Thread-buyer-key (acceptable when the downstream protocol supports it).** The seller passes the buyer's `idempotency_key` (or a deterministic seller-side derivative — e.g., `HMAC(seller_secret, idempotency_key || downstream_provider)`, to avoid leaking the buyer's key across trust boundaries) as the downstream's own idempotency key. The downstream's at-most-once guarantee then covers the case the seller's local persistence missed. The seller MUST still write a claim row on the success path so the cached response can be populated correctly, but the downstream itself becomes the source of truth on retry. + + Sellers MUST document which pattern applies to which downstream in their operational runbook. Sellers MUST NOT use a third pattern of "best-effort dedup on downstream response inspection" — comparing the downstream's response payload to a cached fingerprint to decide whether the call already happened — because the downstream's response shape changes across versions and the fingerprint is a synchronization bug waiting to happen. A claim row OR a threaded key. Not pattern-match-on-response. + + The buyer-visible consequence of this rule: when a seller invokes a slow downstream and the buyer retries during the window, the seller's response on the second request is determined by the seller's policy under rule 9 (`IDEMPOTENCY_IN_FLIGHT` or wait-and-replay), not by the downstream's behavior. Buyers do not need to know which downstream is in the path — the seller MUST present a uniform retry surface regardless. + #### Payload equivalence "Equivalent" means **identical canonical JSON form**, not field-by-field semantic comparison. Sellers MUST determine equivalence by hashing the canonical form and comparing hashes. The canonical form is [RFC 8785 JSON Canonicalization Scheme (JCS)](https://www.rfc-editor.org/rfc/rfc8785) — number serialization, key ordering, and escaping all follow JCS §3 normatively. diff --git a/docs/protocol/calling-an-agent.mdx b/docs/protocol/calling-an-agent.mdx index 8eb96a9ce8..9fd021ba84 100644 --- a/docs/protocol/calling-an-agent.mdx +++ b/docs/protocol/calling-an-agent.mdx @@ -26,7 +26,8 @@ Every mutating tool requires an `idempotency_key` (UUID). - **Same key on retry** → server replays the **same response**, byte-for-byte. Use this for transport-level retries (timeout, 5xx, dropped connection). - **Fresh key** → **new operation**, regardless of body. Generating a new UUID because the previous attempt failed is the most common way naïve callers create duplicate media buys. -- **Same key, different body** → server-defined; most agents return the original cached response and ignore the body change. Don't rely on it. +- **Same key, different canonical body** → `IDEMPOTENCY_CONFLICT`. Sellers MUST reject (rule 5 in [security.mdx#idempotency](/docs/building/by-layer/L1/security#idempotency)) — do not silently apply the second body, do not silently replay the first response. +- **Same key while first request still running** → `IDEMPOTENCY_IN_FLIGHT` (rule 9 in [security.mdx#idempotency](/docs/building/by-layer/L1/security#idempotency)). The seller MAY return this code with `error.details.retry_after` instead of blocking. Wait and retry with the **same key** — minting a fresh key on this code turns a safe retry into a double-execution race. For async flows, the replayed response carries the **same `task_id`** so polling continues against the same task instead of forking. diff --git a/scripts/error-code-drift-dispositions.json b/scripts/error-code-drift-dispositions.json index 88ce80f435..1590aa5b46 100644 --- a/scripts/error-code-drift-dispositions.json +++ b/scripts/error-code-drift-dispositions.json @@ -46,6 +46,11 @@ "target_version": "3.1", "note": "Part of field_scopes feature (#3887). Backport requires the whole feature surface, which is 3.1-shaped." }, + "IDEMPOTENCY_IN_FLIGHT": { + "disposition": "held-for-next-minor", + "target_version": "3.1", + "note": "Pairs with security.mdx#idempotency rule 9 (first-insert-wins / concurrent-retry resolution). Lets sellers reject-and-redirect on concurrent retry instead of blocking; buyers retry with the same key after error.details.retry_after. Wire change — held for 3.1." + }, "PAYMENT_TERMS_NOT_SUPPORTED": { "disposition": "held-for-next-minor", "target_version": "3.1", diff --git a/skills/call-adcp-agent/SKILL.md b/skills/call-adcp-agent/SKILL.md index c8ae023e6e..412180122f 100644 --- a/skills/call-adcp-agent/SKILL.md +++ b/skills/call-adcp-agent/SKILL.md @@ -35,7 +35,8 @@ UUID format. The key is your retry-safety guarantee — and the most common way - **Same key on retry → replay.** The server returns the SAME response — same `task_id`, same `media_buy_id`, same shape, byte-for-byte. Use this for transport-level retries (timeout, 5xx, dropped connection). - **Fresh key on retry → NEW operation.** Generating a new UUID because the previous attempt failed is how you double-book. Reuse the key until you've seen a terminal response (success, error, or non-retryable error). -- **Same key, different body → server-defined.** Most agents return the original cached response and ignore the body change. Don't rely on it — pick a fresh key only when you genuinely want a new operation. +- **Same key, different canonical body → `IDEMPOTENCY_CONFLICT`.** Servers MUST reject. Do not silently apply the second body; do not silently replay the first. If your planner re-ran and produced different bytes, the intent changed — mint a new key. +- **Same key while first request still running → `IDEMPOTENCY_IN_FLIGHT`.** Server returns this with `error.details.retry_after` (seconds) when it doesn't want to block. Wait the hint and retry with the **same key** — minting a fresh key here turns a safe retry into a double-execution race. - For async flows, the replayed response carries the **same `task_id`**, so polling continues against the same task instead of forking a duplicate. Required on: `create_media_buy`, `update_media_buy`, `sync_creatives`, `sync_audiences`, `sync_accounts`, `sync_catalogs`, `sync_event_sources`, `sync_plans`, `sync_governance`, `activate_signal`, `acquire_rights`, `log_event`, `report_usage`, `provide_performance_feedback`, `report_plan_outcome`, `create_property_list`, `update_property_list`, `delete_property_list`, `create_collection_list`, `update_collection_list`, `delete_collection_list`, `create_content_standards`, `update_content_standards`, `calibrate_content`, `si_initiate_session`, `si_send_message`. diff --git a/static/compliance/source/test-kits/parallel-dispatch-runner.yaml b/static/compliance/source/test-kits/parallel-dispatch-runner.yaml new file mode 100644 index 0000000000..db8bb7e5b7 --- /dev/null +++ b/static/compliance/source/test-kits/parallel-dispatch-runner.yaml @@ -0,0 +1,196 @@ +# Parallel Dispatch Runner — Harness Contract Test Kit +# +# Applies to: +# - universal/idempotency.yaml (concurrent-retry phase — verifies the +# first-insert-wins rule under L1/security.mdx#idempotency rule 9 by firing +# two requests with the same idempotency_key in parallel and grading the +# outcome). +# - Any future storyboard that needs to verify behavior under concurrent +# dispatch (e.g., racing webhook deliveries, racing sync calls). +# +# Storyboards are sequential by default — steps run one after another in YAML +# order, with each step's response observed before the next is dispatched. +# That model cannot exercise rule 9 (concurrent retry / first-insert-wins), +# because the second request would always arrive after the first's response +# has been observed. +# +# This test-kit defines the coordination contract between a runner that +# supports firing N requests in parallel and storyboards that need that +# capability. The runner's parallel dispatch happens at the @adcp/client +# layer (or its Python/Go equivalents) using the SDK's batch-call primitive, +# NOT at the YAML-step boundary — the storyboard author MUST NOT try to +# express parallelism via separate sequential steps. + +id: parallel_dispatch_runner +applies_to: + universals: + - idempotency + +description: | + Coordination contract between a runner that supports parallel request + dispatch and a storyboard step that needs to exercise concurrent-retry + semantics. The runner fires N >= 2 logically-simultaneous requests (same + `idempotency_key`, same canonical payload) and observes all responses, then + asserts on the cross-response invariants the storyboard step declares. + +endpoint_scope: sandbox +# Concurrent-retry tests on storyboards that drive spend-committing operations +# (create_media_buy in particular) would create real commitments on retry- +# replay-success paths. Graders MUST target a sandbox/staging endpoint. Agents +# claiming the specialism SHOULD expose a dedicated grading endpoint. + +harness_mode: black_box + +# --- Parallelism mechanism --- +# +# Two viable shapes with different observation properties. The runner selects +# per run mode; storyboard authors SHOULD NOT assume a specific mode. +# +# process_local (default for lint/fast runs): +# The runner calls the SDK's batch-call primitive (Promise.all in JS, +# asyncio.gather in Python, sync.WaitGroup in Go) to dispatch N requests +# from the same process before awaiting any of their responses. +# Inter-request scheduling is at the SDK/transport level, NOT at the YAML- +# step level. Sufficient to exercise the seller's INSERT race; not +# sufficient to exercise true network-level interleaving (the runner's +# event loop still drains in deterministic order under most schedulers). +# +# distributed (optional, full-conformance runs): +# The runner dispatches the N requests from N independent worker processes +# (or N independent CI shards) using a barrier-wait synchronization point. +# All workers block on the barrier and unblock simultaneously, producing +# true network-level concurrency. Materially more expensive operationally; +# reserved for AdCP Verified grading and rule-9-specific deep checks. +# +# The specialism grades observable seller behavior; the dispatch mechanism is +# an implementation detail below the conformance seam. A seller that passes +# under process_local MUST also pass under distributed for the same storyboard +# — any divergence is a conformance bug in the runner, not a legitimate +# seller-behavior difference. + +# --- Step-level coordination keys --- +# +# A storyboard step requiring this contract declares: +# +# requires_contract: parallel_dispatch_runner +# parallel_dispatch: +# count: 2 # required, integer >= 2, <= 10 +# same_idempotency_key: true # required for rule-9 tests; future +# # non-idempotency parallel tests MAY +# # set this false +# barrier_timeout_ms: 5000 # optional; default 5000ms — how long the +# # runner waits at the barrier before +# # firing requests anyway (covers slow +# # distributed workers) +# +# The runner injects identical sample_request bodies on all N dispatches — +# specifically including the same expansion of any `$generate:uuid_v4#alias` +# placeholder, so the seller sees the same canonical hash on every replica. +# Distinct correlation_ids per dispatch ARE allowed (and recommended) because +# `context` is excluded from the canonical hash (see L1/security.mdx Payload +# equivalence) — using distinct correlation_ids lets the runner trace which +# response came from which dispatch in its output. + +parallel_dispatch_step_contract: + required_step_fields: + - parallel_dispatch.count + - parallel_dispatch.same_idempotency_key + optional_step_fields: + - parallel_dispatch.barrier_timeout_ms + count_min: 2 + count_max: 10 + # 10 is a soft ceiling: every runner mode (process_local, distributed) MUST + # support up to 10 parallel dispatches. Storyboards needing higher fan-out + # SHOULD justify it in a step comment; the runner MAY reject counts above + # 10 to bound test runtime. + +# --- Validation primitives --- +# +# Steps requiring parallel dispatch declare validations against the array of +# responses, not a single response. The storyboard step's validations[] block +# acquires two cross-response check kinds that are not available in +# sequential-dispatch steps: +# +# - check: cross_response_field_equal +# path: "media_buy_id" +# description: | +# All N responses (including IDEMPOTENCY_IN_FLIGHT-then-retried +# responses, resolved to their eventual cached response) MUST carry the +# same value at the named path. A divergence is a rule-9 violation — +# the seller produced two distinct resources for the same key. +# +# - check: cross_response_count_distinct +# path: "media_buy_id" +# allowed_values: [1] +# description: | +# The set of distinct values observed at the named path across all N +# responses MUST have cardinality equal to one of allowed_values. For +# rule-9 tests, allowed_values: [1] — exactly one resource was created. +# +# Standard per-response checks (response_schema, field_present, error_code, +# etc.) apply to each response individually; the runner runs them N times +# and aggregates pass/fail per the storyboard's existing severity rules. + +cross_response_check_kinds: + - cross_response_field_equal + - cross_response_count_distinct + +# --- Resolution of IDEMPOTENCY_IN_FLIGHT in cross-response checks --- +# +# A seller adopting the reject-and-redirect policy under rule 9 may return +# IDEMPOTENCY_IN_FLIGHT on the second (or later) dispatched request rather +# than wait-and-replay. The runner MUST resolve IDEMPOTENCY_IN_FLIGHT +# responses to their eventual cached response before applying cross-response +# checks: when the runner observes IDEMPOTENCY_IN_FLIGHT, it waits the +# `error.details.retry_after` seconds and retries the SAME idempotency_key +# (per the buyer-side normative behavior). The cross-response check then +# operates on the resolved response set, not the raw initial set. +# +# Maximum total wall-time for one in-flight resolution is bounded by the +# runner's overall step-timeout budget. Sellers returning IDEMPOTENCY_IN_FLIGHT +# with a retry_after that exceeds the runner's budget will fail the step on +# timeout — graded as `step_timeout`, not `rule_9_violation`. Sellers tuning +# retry_after for production traffic SHOULD test their values against the +# runner's documented timeout (default 60s; CI configurable). + +# --- What the contract does NOT cover --- +# +# Network-level packet timing / ordering. process_local mode delivers requests +# in deterministic event-loop order; distributed mode introduces real +# interleaving but does not synthesize adversarial packet schedules. Sellers +# whose correctness depends on adversarial network conditions need additional +# fault-injection harnesses outside this contract. +# +# Multi-account / multi-agent racing. The contract dispatches N requests +# within a single (authenticated_agent, account) scope — the same scope as +# the idempotency key itself. Storyboards needing to exercise cross-scope +# races (e.g., agency-vs-direct-buyer collisions on a shared natural key) +# require a separate contract. +# +# Crash injection. The contract does not interrupt the seller mid-request. +# Storyboards testing seller behavior under crashes / restarts / failovers +# need a separate fault-injection harness — out of scope here. + +# --- Reviewer checks --- +# +# Some properties of rule 9 cannot be programmatically graded by this +# contract and MUST be confirmed by reviewer inspection of the seller's +# implementation and runbook: +# +# - The seller's claim-row / unique-constraint pattern actually backs the +# observed first-insert-wins behavior (not a happy-path race that +# coincidentally avoided the conflict in the test run). +# - The seller's downstream reconciliation pattern (rule 10) is documented +# in the operational runbook. The contract observes only the buyer- +# visible response surface; the downstream-side behavior is not directly +# observable from a black-box harness. +# - The seller's in-flight detection does NOT leak across scope boundaries +# — an attacker authenticated as agent B with a stolen key from agent A +# receives the same response shape and timing whether agent A's row is +# in flight, completed, or never existed. Verified by manual probing, +# not by this contract. + +reviewer_checks: + - "Confirm the seller's claim-row / unique-constraint implementation backs the observed first-insert-wins behavior, rather than a happy-path race the test happened to avoid (review code or runbook describing the INSERT … ON CONFLICT pattern on the scope tuple)." + - "Confirm the seller's downstream-reconciliation pattern (rule 10) is documented in their operational runbook: which downstream calls use write-claim-before-invoke and which thread the buyer-key (or a derivative)." + - "Confirm the seller's in-flight state does NOT leak across (authenticated_agent, account_id) scope boundaries — probe with a stolen key from a different scope and verify the response shape and timing match the never-seen-key path." diff --git a/static/compliance/source/universal/idempotency.yaml b/static/compliance/source/universal/idempotency.yaml index 5a0b4bd926..74609d97be 100644 --- a/static/compliance/source/universal/idempotency.yaml +++ b/static/compliance/source/universal/idempotency.yaml @@ -2,7 +2,7 @@ id: idempotency version: "1.0.0" title: "Idempotency enforcement" category: core -summary: "Validates that mutating requests enforce idempotency_key — replays return cached responses, key reuse with a different payload returns IDEMPOTENCY_CONFLICT, and fresh keys create new resources." +summary: "Validates that mutating requests enforce idempotency_key — replays return cached responses, key reuse with a different payload returns IDEMPOTENCY_CONFLICT, fresh keys create new resources, and concurrent retries with the same key produce exactly one resource (first-insert-wins under rule 9)." track: core # Cross-step assertions (adcontextprotocol/adcp#2639). These convert two @@ -33,7 +33,7 @@ invariants: narrative: | Every mutating request in AdCP carries an idempotency_key so buyers can safely retry - after network errors without double-booking. This storyboard walks through the four + after network errors without double-booking. This storyboard walks through the observable behaviors a seller MUST implement: 1. First call with a fresh key is processed normally and returns the canonical response. @@ -55,6 +55,12 @@ narrative: | key_reuse_conflict. An end-to-end phase that drives a deterministic terminal error and replays the key is deferred pending a generic force-error controller verb (see adcontextprotocol/adcp#2760). + 6. Concurrent retries with the same key produce exactly one resource. The seller + MAY wait-and-replay or return IDEMPOTENCY_IN_FLIGHT — both are conformant under + security.mdx#idempotency rule 9 (first-insert-wins) as long as the resolved + response set converges on a single media_buy_id. Graded via the + parallel_dispatch_runner contract; runners without parallel-dispatch skip + this phase with a stable not_applicable marker. Missing idempotency_key on any mutating request MUST be rejected with INVALID_REQUEST. Sellers MUST declare adcp.idempotency on get_adcp_capabilities — either @@ -565,6 +571,117 @@ phases: value: "idempotency--create_media_buy_fresh_key" description: "Context correlation_id returned unchanged" + - id: concurrent_retry + title: "Concurrent retries with the same key produce exactly one resource" + narrative: | + A buyer's transport timeout fires before the seller's downstream call + returns; the buyer retries with the same idempotency_key while the + first request is still executing. Per L1/security.mdx#idempotency + rule 9 (first-insert-wins), the seller MUST resolve the race + deterministically — exactly one resource is created, and both buyer + requests converge on the same media_buy_id. + + The seller picks one of two policies and MUST behave consistently: + + - **Wait-and-replay**: the second request blocks until the first + completes, then returns the cached response with `replayed: true`. + - **Reject-and-redirect**: the second request returns + `IDEMPOTENCY_IN_FLIGHT` immediately with + `error.details.retry_after`. The buyer retries with the same key + after the hint elapses and receives the cached response. + + This phase exercises both policies via the parallel_dispatch_runner + contract: it fires two requests with the same fresh idempotency_key + and the same canonical payload, and asserts that the resolved + response set contains exactly one distinct media_buy_id. The runner + transparently resolves IDEMPOTENCY_IN_FLIGHT responses by retrying + with the same key after `retry_after` elapses, so the cross-response + check operates on the resolved set regardless of which policy the + seller adopted. + + Runners without parallel-dispatch support skip this phase with a + stable not_applicable marker. The phase does not regress the + sequential idempotency contract — sellers that fail this phase due + to runner absence are still gradeable on the rest of the storyboard. + + steps: + - id: create_media_buy_concurrent + title: "Two parallel create_media_buy calls with same key converge on one resource" + narrative: | + Fire two create_media_buy requests with the same fresh UUID v4 + idempotency_key and byte-identical canonical payloads. The + runner dispatches them via its parallel-dispatch primitive (SDK + batch-call or distributed barrier), observes both responses, + resolves any IDEMPOTENCY_IN_FLIGHT to its eventual cached + response, and applies the cross-response checks below. + + Programmatic verification of "exactly one resource created" runs + via `cross_response_count_distinct` on `media_buy_id`. A seller + that ignored rule 9 (re-executed both, two media buys created) + would produce cardinality 2 here and fail; a seller that + correctly serializes via first-insert-wins produces cardinality + 1 regardless of which policy (wait-and-replay or + reject-and-redirect) it adopted. + + The accompanying `cross_response_field_equal` check is redundant + when cardinality is 1, but documents the equivalence intent + directly: every dispatch's resolved response carries the same + media_buy_id. + task: create_media_buy + schema_ref: "media-buy/create-media-buy-request.json" + response_schema_ref: "media-buy/create-media-buy-response.json" + doc_ref: "/media-buy/task-reference/create_media_buy" + comply_scenario: idempotency_concurrent + stateful: true + requires_contract: parallel_dispatch_runner + parallel_dispatch: + count: 2 + same_idempotency_key: true + barrier_timeout_ms: 5000 + expected: | + After resolving any IDEMPOTENCY_IN_FLIGHT responses to their + eventual cached response, both dispatches' responses carry the + same media_buy_id. Exactly one media buy was created. The + resolved-response set has cardinality 1 on media_buy_id. + + A seller adopting wait-and-replay returns the same response on + both dispatches directly. A seller adopting reject-and-redirect + returns the cached response on one dispatch and + IDEMPOTENCY_IN_FLIGHT on the other; the runner resolves the + latter by retrying with the same key after `retry_after` and + observes the cached response. Either path is conformant. + + sample_request: + idempotency_key: "$generate:uuid_v4#concurrent_key" + account: + brand: + domain: "acmeoutdoor.example" + operator: "pinnacle-agency.example" + brand: + domain: "acmeoutdoor.example" + start_time: "2026-07-01T00:00:00Z" + end_time: "2026-07-31T23:59:59Z" + packages: + - product_id: "test-product" + budget: 5000 + pricing_option_id: "test-pricing" + + context: + correlation_id: "idempotency--create_media_buy_concurrent" + validations: + - check: cross_response_count_distinct + path: "media_buy_id" + allowed_values: [1] + description: "Concurrent retries produced exactly one distinct media_buy_id across both dispatches (rule 9: first-insert-wins)" + - check: cross_response_field_equal + path: "media_buy_id" + description: "Both dispatches' resolved responses carry the same media_buy_id" + + reviewer_checks: + - "Confirm the seller's claim-row / unique-constraint implementation backs the observed first-insert-wins behavior, rather than a happy-path race the test happened to avoid (review code or runbook describing the INSERT … ON CONFLICT pattern on the scope tuple)." + - "If the seller returned IDEMPOTENCY_IN_FLIGHT on the second dispatch, confirm `error.details.retry_after` was populated with a plausible value (seconds, integer, > 0) and that the cached response was available within the hint window. A seller that consistently overshoots retry_after produces brittle retry timing for buyers." + - "Confirm the seller's in-flight detection does NOT leak across (authenticated_agent, account_id) scope boundaries — probe with a stolen key from a different scope and verify the response shape and timing match the never-seen-key path." + - id: verify_media_buy_count title: "Verify dedup actually happened" narrative: | diff --git a/static/compliance/source/universal/runner-output-contract.yaml b/static/compliance/source/universal/runner-output-contract.yaml index d93da8a6d9..a987b218af 100644 --- a/static/compliance/source/universal/runner-output-contract.yaml +++ b/static/compliance/source/universal/runner-output-contract.yaml @@ -59,6 +59,8 @@ authored_check_kinds: - field_less_than - field_equals_context - upstream_traffic + - cross_response_field_equal + - cross_response_count_distinct # Conforming implementation: adcp-client PR #611 # (src/lib/testing/storyboard/types.ts — RunnerRequestRecord, diff --git a/static/schemas/source/enums/error-code.json b/static/schemas/source/enums/error-code.json index a94d447053..a703bdac61 100644 --- a/static/schemas/source/enums/error-code.json +++ b/static/schemas/source/enums/error-code.json @@ -31,6 +31,7 @@ "CONFLICT", "IDEMPOTENCY_CONFLICT", "IDEMPOTENCY_EXPIRED", + "IDEMPOTENCY_IN_FLIGHT", "CREATIVE_DEADLINE_EXCEEDED", "INVALID_STATE", "MEDIA_BUY_NOT_FOUND", @@ -97,6 +98,7 @@ "CONFLICT": "Concurrent modification detected. The resource was modified by another request between read and write. Recovery: transient (re-read the resource and retry with current state).", "IDEMPOTENCY_CONFLICT": "An earlier request with the same idempotency_key was processed with a different canonical payload within the seller's replay window. Distinct from CONFLICT (concurrent write) — this indicates the client reused a key across semantically different requests. Recovery: correctable (use a fresh UUID v4 for the new request, or resend the exact original payload to get the cached response).", "IDEMPOTENCY_EXPIRED": "The idempotency_key was seen previously but its cached response has been evicted because it is past the seller's declared replay_ttl_seconds. Distinct from IDEMPOTENCY_CONFLICT (different payload within window) — this indicates the retry arrived too late for at-most-once guarantees. Recovery: correctable (perform a natural-key check — e.g., get_media_buys by context.internal_campaign_id — to determine whether the original request succeeded, then either accept that result or generate a fresh idempotency_key for a new attempt). If the buyer has any evidence the prior call succeeded (partial response received before crash, entry in the buyer's own DB, a webhook fired), the buyer MUST do the natural-key check BEFORE minting a new key — minting a new key in that situation is exactly how double-creation happens.", + "IDEMPOTENCY_IN_FLIGHT": "A prior request with the same `idempotency_key` is still being processed and has not yet produced a cached response. The second request arrived before the first completed. Sellers MAY return this code instead of blocking the second caller until the first finishes — useful when the first call invokes a slow downstream system (SSP, ad server, payment provider). Distinct from IDEMPOTENCY_CONFLICT (different canonical payload — a client bug) and from CONFLICT (concurrent modification of a different resource) — IDEMPOTENCY_IN_FLIGHT is the seller telling the buyer 'your retry was correct but your previous attempt is still running, come back shortly.' Sellers SHOULD populate `error.details.retry_after` (seconds, integer) with a wait hint based on the first request's elapsed time and expected completion. Buyers MUST treat this as transient and MUST NOT mint a fresh `idempotency_key` — minting a new key turns a safe retry into a double-execution race. Recovery: transient (wait `retry_after` seconds and retry with the same `idempotency_key`; the second attempt will either replay the now-cached response or, if still in flight, return IDEMPOTENCY_IN_FLIGHT again).", "INVALID_STATE": "Operation is not permitted for the resource's current status (e.g., updating a completed or canceled media buy, or modifying a canceled package). Recovery: correctable (check current status via get_media_buys and adjust request).", "MEDIA_BUY_NOT_FOUND": "Referenced media buy does not exist or is not accessible to the requesting agent. Recovery: correctable (verify media_buy_id or buyer_ref).", "NOT_CANCELLABLE": "The media buy or package cannot be canceled in its current state. The seller may have contractual or operational constraints that prevent cancellation. Recovery: correctable (check the seller's cancellation policy or contact the seller).", @@ -240,6 +242,10 @@ "recovery": "correctable", "suggestion": "perform a natural-key check to determine whether the original request succeeded; if no evidence of success, generate a fresh idempotency_key for a new attempt" }, + "IDEMPOTENCY_IN_FLIGHT": { + "recovery": "transient", + "suggestion": "wait error.details.retry_after seconds and retry with the SAME idempotency_key — MUST NOT mint a fresh key (turns a safe retry into a double-execution race)" + }, "CREATIVE_DEADLINE_EXCEEDED": { "recovery": "correctable", "suggestion": "check creative_deadline via get_media_buys before submitting changes, or negotiate a deadline extension with the seller" From 404fec73228b3312ffc38e0855a10ab18291edb6 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 11 May 2026 10:34:25 -0400 Subject: [PATCH 2/3] =?UTF-8?q?spec(idempotency):=20address=20expert=20rev?= =?UTF-8?q?iew=20=E2=80=94=20separate=20cross-response=20check=20kinds,=20?= =?UTF-8?q?bound=20in-flight=20TTL,=20tighten=20rule=2010=20reconciliation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expert review (run-by-experts skill, 4 reviewers in parallel) surfaced 3 convergent blockers and 4 single-reviewer must-fix items on the rule-9/10 spec text and storyboard surface. All addressed: CONVERGENT (2+ reviewers): - Cross-response check kinds (cross_response_field_equal, cross_response_count_distinct) moved out of authored_check_kinds into a sibling cross_response_check_kinds enum in runner-output-contract.yaml. Prevents the per-response-interpretation footgun. Lint updated to read both arrays. Storyboard-schema.yaml gains a dedicated section documenting parallel_dispatch step block + the two check kinds. Runner-output-contract.yaml expected/actual blocks document both kinds' payload shapes. - In-flight TTL bound: rule 9 now requires sellers to bound in-flight row lifetime to their declared per-task handler timeout and release the row on timeout. Also caps retry_after at <= replay_ttl_seconds — recommends an order of magnitude below the TTL ceiling. - Rule 10 reviewer-graded note: explicit clause that rule 10 conformance is reviewer-graded, not programmatically graded by the storyboard suite. parallel_dispatch_runner already lists rule-10 as a reviewer_check; this surfaces it in the spec text too. SINGLE-REVIEWER MUST-FIX: - Rule 9 hash-availability: spec text now requires sellers to persist the canonical payload hash on INSERT (not a sentinel). Closes the gap where the SDK store's __adcp_in_flight__ placeholder prevented the rule-9-vs-rule-5 boundary from being detectable. - Rule 10 HMAC derivation rationale: tightened to per-downstream- provider derivation (HMAC(K_provider, idempotency_key)) rather than a single shared seller secret across all downstreams. Prevents cross-provider replay if any single downstream is compromised. - Rule 10 downstream-error key strip: sellers MUST NOT propagate the buyer's idempotency_key (or a reversible derivative) in error envelopes originating from a downstream. Closes a cross-trust- boundary key-disclosure surface. - Rule 10 claim-row-without-invoke: if the downstream reports no record of downstream_request_id (claim row persisted but seller crashed pre-invoke), the seller MUST proceed with the invocation rather than fail-closed. - Storyboard verify_media_buy_count narrative: corrected from "only TWO media buys" claim (false after the new concurrent_retry phase creates a third) to "captured media_buy_ids both exist and are distinct" — matches what the step's query actually asserts. Plus a transitional note in the changeset clarifying that SERVICE_UNAVAILABLE + retry_after on the in-flight branch remains conformant under wait-and-replay; IDEMPOTENCY_IN_FLIGHT is only required for reject-and-redirect. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/idempotency-rule-9-and-10.md | 2 + docs/building/by-layer/L1/security.mdx | 20 ++++-- scripts/lint-storyboard-check-enum.cjs | 7 +- .../source/universal/idempotency.yaml | 25 ++++--- .../universal/runner-output-contract.yaml | 47 ++++++++++++- .../source/universal/storyboard-schema.yaml | 67 +++++++++++++++++++ 6 files changed, 150 insertions(+), 18 deletions(-) diff --git a/.changeset/idempotency-rule-9-and-10.md b/.changeset/idempotency-rule-9-and-10.md index b9988f754b..86c43f03af 100644 --- a/.changeset/idempotency-rule-9-and-10.md +++ b/.changeset/idempotency-rule-9-and-10.md @@ -12,6 +12,8 @@ Two new normative rules in `L1/security.mdx#idempotency`: **New error code: `IDEMPOTENCY_IN_FLIGHT`** (held for 3.1 per the wire-stability policy). Recovery: transient. Buyers MUST retry with the **same** `idempotency_key` after `error.details.retry_after` — minting a fresh key on this code turns a safe retry into a double-execution race. +**Transitional note on `SERVICE_UNAVAILABLE + retry_after`.** Both reference implementations today (the Python sales-agent at `wonderstruck.sales-agent.scope3.com` and the `@adcp/sdk` middleware) implement wait-and-replay (rule 9's other policy) and never need to emit `IDEMPOTENCY_IN_FLIGHT`. SDKs that previously emitted `SERVICE_UNAVAILABLE + retry_after: 1` on the in-flight branch are NOT out of compliance with rule 9 as long as they adopt wait-and-replay end-to-end — `IDEMPOTENCY_IN_FLIGHT` is only required when a seller picks reject-and-redirect. The `@adcp/sdk` middleware swap from `SERVICE_UNAVAILABLE` to `IDEMPOTENCY_IN_FLIGHT` is tracked separately (adcp-client follow-up); it's a wire-code tightening, not a behavioral change. + **Storyboard coverage.** `static/compliance/source/universal/idempotency.yaml` gains a `concurrent_retry` phase using two new cross-response check kinds (`cross_response_count_distinct`, `cross_response_field_equal`) that operate on the resolved response set across N parallel dispatches. The runner contract is documented in the new `test-kits/parallel-dispatch-runner.yaml`; runners without parallel-dispatch support skip the phase with a stable not_applicable marker. SDK/runner implementation tracked separately (adcp-client follow-up). Author skill (`skills/call-adcp-agent/SKILL.md`) and the buyer-facing `docs/protocol/calling-an-agent.mdx` updated so buyers know to wait-and-retry on `IDEMPOTENCY_IN_FLIGHT` rather than mint a fresh key. diff --git a/docs/building/by-layer/L1/security.mdx b/docs/building/by-layer/L1/security.mdx index dc1cffee23..661c4c684b 100644 --- a/docs/building/by-layer/L1/security.mdx +++ b/docs/building/by-layer/L1/security.mdx @@ -299,24 +299,34 @@ This section applies only to AdCP task requests. OpenRTB bid streams have their The ceiling is per `(authenticated_agent, account)` — the same scope as the idempotency key itself (bullet 1) — so a multi-account agency does not have its per-account budgets collapsed into a single shared quota. `RATE_LIMITED` rejections MUST populate `retry_after` (seconds) per the [error handling taxonomy](/docs/building/by-layer/L3/error-handling#rate-limit-handling) and MUST NOT be cached as idempotency responses (rule 3: only successful responses are cached). Sellers SHOULD enforce `retry_after` as a cheap rejection floor — a buyer retrying before `retry_after` elapses SHOULD hit a pre-auth token bucket (e.g., at a reverse-proxy layer) rather than re-entering the full schema-validate-and-cache-check pipeline on every retry. Without this discipline, misbehaving buyers can amplify load on the rate-limiter itself. -9. **Concurrent retries — first-insert-wins.** A second request carrying the same `(authenticated_agent, account_id, idempotency_key)` MAY arrive while the first request is still executing — most commonly when the buyer's transport timeout fires before the seller's downstream call returns, and the buyer retries. Sellers MUST resolve the race deterministically; they MUST NOT execute the side effect twice and MUST NOT silently drop the second request. Resolution is a `(unique constraint, INSERT … ON CONFLICT DO NOTHING)` pattern on the scope tuple: the first row to land owns execution; subsequent requests observe an existing row whose response slot is not yet populated. +9. **Concurrent retries — first-insert-wins.** A second request carrying the same `(authenticated_agent, account_id, idempotency_key)` MAY arrive while the first request is still executing — most commonly when the buyer's transport timeout fires before the seller's downstream call returns, and the buyer retries. Sellers MUST resolve the race deterministically; they MUST NOT execute the side effect twice and MUST NOT silently drop the second request. Resolution is a `(unique constraint, INSERT … ON CONFLICT DO NOTHING)` pattern on the scope tuple: the first row to land owns execution and stores the canonical payload hash on the in-flight row (NOT a sentinel); subsequent requests observe an existing row whose response slot is not yet populated but whose payload hash IS populated. Sellers MUST handle the second request by one of two policies and MUST behave consistently across calls — clients infer the policy from the first response within a session and apply it to subsequent retries: - **Wait-and-replay** (preferred for fast operations, <5s typical): the seller blocks the second request until the first completes, then returns the cached response with `replayed: true`. Total wall-time for the second call is bounded by the seller's request-timeout budget. - **Reject-and-redirect** (preferred for slow operations involving long-running downstream calls): the seller returns `IDEMPOTENCY_IN_FLIGHT` immediately, with `error.details.retry_after` (seconds, integer) populated based on the first request's elapsed time and expected completion. Buyers MUST retry with the same `idempotency_key` after the hint elapses — a buyer that mints a fresh key on `IDEMPOTENCY_IN_FLIGHT` turns a safe retry into the exact double-execution race this rule prevents. - A second request with the same key AND a *different* canonical payload during the in-flight window MUST return `IDEMPOTENCY_CONFLICT` (rule 5), not `IDEMPOTENCY_IN_FLIGHT` — the canonical-form mismatch is computable at INSERT time against the row's stored hash, so the conflict is detectable without waiting for the first request's response. Per rule 3, if the first request ultimately fails (validation error, downstream timeout, internal error), the `(in_flight)` row is released — the key returns to "never seen" state and a subsequent retry re-executes from scratch. Sellers MUST NOT leak the in-flight state across the scope boundary: an attacker probing a candidate key MUST receive the same response shape and timing whether the row exists, is in flight, or has never existed. + A second request with the same key AND a *different* canonical payload during the in-flight window MUST return `IDEMPOTENCY_CONFLICT` (rule 5), not `IDEMPOTENCY_IN_FLIGHT` — the canonical-form mismatch is computable at INSERT time against the row's stored hash, so the conflict is detectable without waiting for the first request's response. Sellers whose backing store cannot persist the real canonical hash until the handler completes (e.g., a placeholder-sentinel pattern) MUST upgrade the store to persist the hash at INSERT time before declaring rule 9 conformance — the alternative (returning `IDEMPOTENCY_IN_FLIGHT` on a same-key-different-payload race and only surfacing the conflict after the first request completes) silently delays detection of a real client bug. + + Per rule 3, if the first request ultimately fails (validation error, downstream timeout, internal error), the `(in_flight)` row is released — the key returns to "never seen" state and a subsequent retry re-executes from scratch. Sellers MUST bound the lifetime of an in-flight row to their declared per-task handler timeout, and MUST release the row (treat as failed per rule 3) when that timeout fires — even if the downstream has not yet responded. Without this bound, a hung handler indefinitely returns `IDEMPOTENCY_IN_FLIGHT` for the same key, locking the buyer out of any safe retry path. + + Sellers using reject-and-redirect MUST set `error.details.retry_after` to a value no greater than `replay_ttl_seconds` (declared in `capabilities.idempotency`). A buyer instructed to wait past the seller's own replay window is being told to wait until the response can no longer be replayed — the wait is vacuous and the buyer either ends up minting a fresh key (the failure mode this rule prevents) or hits `IDEMPOTENCY_EXPIRED` on retry. Recommended bound: an order of magnitude below the replay TTL, derived from the seller's typical handler latency rather than the TTL ceiling. + + Sellers MUST NOT leak the in-flight state across the scope boundary: an attacker probing a candidate key MUST receive the same response shape and timing whether the row exists, is in flight, or has never existed. 10. **Crossing service boundaries — downstream reconciliation.** Sellers commonly invoke downstream systems during request handling — SSP/ad-server calls on `create_media_buy`, payment-provider calls on billing operations, governance-agent calls on `check_governance`. These calls have their own failure modes that can leave the seller in a "downstream unknown" state: the network connection dropped after the downstream accepted the request but before its response arrived; the seller process crashed mid-call; a region failover swapped the worker before the response was persisted. Rule 3 (only successful responses cached) is necessary but not sufficient: a seller that simply doesn't cache and re-executes on retry will double-invoke the downstream and create duplicate side effects there. - Sellers MUST adopt one of two reconciliation patterns for every downstream call whose duplicate-invocation has business consequences (resource creation, payment movement, irreversible state change). Read-only downstream calls are exempt. + **Conformance grading.** This rule is reviewer-graded, not programmatically graded by the compliance storyboard suite. Black-box observation cannot distinguish "the seller has a claim row" from "the seller got lucky on the test run." The `parallel_dispatch_runner` test-kit lists rule-10 conformance under `reviewer_checks` — sellers attesting to rule-10 conformance MUST surface their operational runbook describing which pattern applies to which downstream, and reviewers verify the implementation against that runbook. The other normative rules (1–9) are programmatically graded. - - **Write-claim-before-invoke (preferred default).** Before invoking the downstream, the seller persists a "claim" row in the same transaction as the idempotency cache row — typically `{idempotency_key, downstream_provider, downstream_request_id, status: 'invoked', invoked_at}` — using the seller-generated `downstream_request_id` it will pass to the downstream as the downstream's own correlation/idempotency identifier. On retry, before invoking the downstream again, the seller MUST look up the claim row by `(idempotency_key, downstream_provider)` and reconcile: query the downstream by `downstream_request_id` to determine the true outcome, then resume cache population from there. The seller MUST NOT treat a missing local record as "downstream call did not happen" — a crash between downstream-accepts and local-persist is exactly the case where it did happen and the local record is missing. - - **Thread-buyer-key (acceptable when the downstream protocol supports it).** The seller passes the buyer's `idempotency_key` (or a deterministic seller-side derivative — e.g., `HMAC(seller_secret, idempotency_key || downstream_provider)`, to avoid leaking the buyer's key across trust boundaries) as the downstream's own idempotency key. The downstream's at-most-once guarantee then covers the case the seller's local persistence missed. The seller MUST still write a claim row on the success path so the cached response can be populated correctly, but the downstream itself becomes the source of truth on retry. + Sellers MUST adopt one of two reconciliation patterns for every downstream call whose duplicate-invocation has business consequences (resource creation, payment movement, irreversible state change). Read-only downstream calls (cache lookups, eligibility checks that don't write) are exempt — but borderline cases like fraud-scoring lookups that also write to a downstream audit log count as writes for this rule (the audit log entry is the side effect). + + - **Write-claim-before-invoke (preferred default).** Before invoking the downstream, the seller persists a "claim" row in the same transaction as the idempotency cache row — typically `{idempotency_key, downstream_provider, downstream_request_id, status: 'invoked', invoked_at}` — using the seller-generated `downstream_request_id` it will pass to the downstream as the downstream's own correlation/idempotency identifier. On retry, before invoking the downstream again, the seller MUST look up the claim row by `(idempotency_key, downstream_provider)` and reconcile: query the downstream by `downstream_request_id` to determine the true outcome, then resume cache population from there. The seller MUST NOT treat a missing local record as "downstream call did not happen" — a crash between downstream-accepts and local-persist is exactly the case where it did happen and the local record is missing. If the downstream reports no record of `downstream_request_id` (the claim row was persisted but the seller crashed before invoking), the seller MUST treat the call as not-yet-invoked and proceed with the invocation; the claim row already reserves the `downstream_request_id`, so the downstream's own idempotency will dedup any subsequent retry. + - **Thread-buyer-key (acceptable when the downstream protocol supports it).** The seller passes a per-downstream-provider derivative of the buyer's `idempotency_key` as the downstream's own idempotency key — typically `HMAC(K_provider, idempotency_key)` where `K_provider` is derived from the seller's KMS-managed root keyed by provider identity (one key per downstream, not one shared seller secret across all downstreams). Per-provider derivation prevents cross-provider replay if any single downstream is compromised; a shared seller secret across all downstreams collapses every provider into a single key-exposure blast radius. The downstream's at-most-once guarantee then covers the case the seller's local persistence missed. The seller MUST still write a claim row on the success path so the cached response can be populated correctly, but the downstream itself becomes the source of truth on retry. The seller MUST NOT pass the buyer's raw `idempotency_key` to a downstream — the buyer's key is a capability token within its TTL (see "Keys are security-sensitive" below) and forwarding it across the seller-downstream trust boundary widens the capability surface. Sellers MUST document which pattern applies to which downstream in their operational runbook. Sellers MUST NOT use a third pattern of "best-effort dedup on downstream response inspection" — comparing the downstream's response payload to a cached fingerprint to decide whether the call already happened — because the downstream's response shape changes across versions and the fingerprint is a synchronization bug waiting to happen. A claim row OR a threaded key. Not pattern-match-on-response. + Sellers MUST NOT include the buyer's `idempotency_key` (or any reversible derivative thereof) in error envelopes returned to the buyer when those errors originated from the downstream. Downstream errors that mention the seller's per-downstream-provider key (or the buyer's key, if the seller incorrectly threaded it raw) MUST be re-keyed or stripped before propagating to the buyer — otherwise a downstream error message becomes a cross-trust-boundary key-disclosure surface. + The buyer-visible consequence of this rule: when a seller invokes a slow downstream and the buyer retries during the window, the seller's response on the second request is determined by the seller's policy under rule 9 (`IDEMPOTENCY_IN_FLIGHT` or wait-and-replay), not by the downstream's behavior. Buyers do not need to know which downstream is in the path — the seller MUST present a uniform retry surface regardless. #### Payload equivalence diff --git a/scripts/lint-storyboard-check-enum.cjs b/scripts/lint-storyboard-check-enum.cjs index 6d6b7bf70a..6d85aaad9b 100644 --- a/scripts/lint-storyboard-check-enum.cjs +++ b/scripts/lint-storyboard-check-enum.cjs @@ -51,14 +51,15 @@ const SYNTHESIZED_CHECK_KINDS = new Set([ function loadAuthoredCheckKinds() { const doc = yaml.load(fs.readFileSync(CONTRACT_FILE, 'utf8')); - const kinds = doc && Array.isArray(doc.authored_check_kinds) ? doc.authored_check_kinds : null; - if (!kinds || kinds.length === 0) { + const authored = doc && Array.isArray(doc.authored_check_kinds) ? doc.authored_check_kinds : null; + if (!authored || authored.length === 0) { throw new Error( `runner-output-contract.yaml is missing the \`authored_check_kinds\` list. ` + `This lint reads that field as the canonical enum; restore it before running.` ); } - return new Set(kinds); + const crossResponse = doc && Array.isArray(doc.cross_response_check_kinds) ? doc.cross_response_check_kinds : []; + return new Set([...authored, ...crossResponse]); } const RULE_MESSAGES = { diff --git a/static/compliance/source/universal/idempotency.yaml b/static/compliance/source/universal/idempotency.yaml index 74609d97be..305df844a2 100644 --- a/static/compliance/source/universal/idempotency.yaml +++ b/static/compliance/source/universal/idempotency.yaml @@ -685,18 +685,25 @@ phases: - id: verify_media_buy_count title: "Verify dedup actually happened" narrative: | - Call get_media_buys to confirm that only TWO media buys were created across - all prior steps — one for the initial + replay pair (same key, deduplicated) - and one for the fresh-key request. If the seller ignored the idempotency_key, - there would be three or more. + Call get_media_buys to confirm the two captured media_buy_ids both exist + and are distinct. `initial_media_buy_id` was created on the first + `create_media_buy` call and reused on the replay (same key, deduplicated); + `fresh_key_media_buy_id` was created on the fresh-key request. The + `concurrent_retry` phase (when graded) also exercises first-insert-wins + but its resource is not queried here — that phase has its own + cross-response cardinality check. If the seller ignored the idempotency_key + on the replay branch, `initial_media_buy_id` would not match across the + replay step's response. steps: - id: get_media_buys_dedup_check - title: "Confirm dedup via get_media_buys" + title: "Confirm captured media_buy_ids exist and are distinct" narrative: | - Query the two media_buy_ids captured from prior steps. Both should exist - and be distinct. This is an end-to-end verification that the replay did - not create a duplicate row. + Query the two captured media_buy_ids from prior steps. Both should + exist (proving fresh-key and initial calls each created a real + resource) and be distinct (proving the fresh-key call was treated as + a new request, not deduplicated against the initial). This is an + end-to-end verification that the replay did not create a duplicate row. task: get_media_buys schema_ref: "media-buy/get-media-buys-request.json" response_schema_ref: "media-buy/get-media-buys-response.json" @@ -704,7 +711,7 @@ phases: comply_scenario: idempotency_verify stateful: true expected: | - Return exactly two media buys with the two captured IDs. Not three. + Return both captured media buys with the two captured IDs, distinct. sample_request: account: diff --git a/static/compliance/source/universal/runner-output-contract.yaml b/static/compliance/source/universal/runner-output-contract.yaml index a987b218af..6980f593e2 100644 --- a/static/compliance/source/universal/runner-output-contract.yaml +++ b/static/compliance/source/universal/runner-output-contract.yaml @@ -59,6 +59,21 @@ authored_check_kinds: - field_less_than - field_equals_context - upstream_traffic + +# Cross-response check kinds — applied across the response set of a +# step that dispatches multiple parallel requests under the +# `parallel_dispatch_runner` contract. These kinds are NOT applicable +# to single-dispatch steps: a runner that applies them per-response +# produces nonsense (a single response trivially has cardinality 1 +# on any field). Runners MUST gate dispatch of these checks on the +# step declaring `requires_contract: parallel_dispatch_runner` and a +# `parallel_dispatch` block; otherwise grade the validation +# `not_applicable` per the forward-compat clause. +# +# The `lint-storyboard-check-enum.cjs` lint reads BOTH this list and +# `authored_check_kinds` as the canonical published-time enum, so +# storyboards declaring either kind pass the typo gate. +cross_response_check_kinds: - cross_response_field_equal - cross_response_count_distinct @@ -123,7 +138,12 @@ validation_result: # http_status_in | error_code | on_401_require_header | # resource_equals_agent_url | any_of | refs_resolve | # a2a_submitted_artifact | field_less_than | - # field_equals_context | upstream_traffic + # field_equals_context | upstream_traffic | + # cross_response_field_equal | + # cross_response_count_distinct (the last two are + # cross-response checks; see cross_response_check_kinds + # above and storyboard-schema.yaml > Cross-response + # assertions for semantics). # # For runner-synthesized grading (not authored in the # storyboard validations array, generated by the runner @@ -199,6 +219,19 @@ validation_result: # identifier_paths, purpose_filter, # since, attestation_mode_required } — # whichever fields the storyboard set. + # cross_response_field_equal + # → object { path } — the JSON path + # asserted equal across every + # resolved response in the step's + # dispatch set. + # cross_response_count_distinct + # → object { path, allowed_values } — + # the path scanned, plus the + # permitted cardinalities of + # distinct values observed at + # that path across the resolved + # response set (typically [1] + # for rule-9 tests). - actual # machine-readable actual value observed: # response_schema → array of schema errors # (each { instance_path, @@ -237,6 +270,18 @@ validation_result: # missing_identifier_values }. # The full recorded_calls array # lives in `response.payload`; + # cross_response_field_equal + # → object { observed_values: [...] } — + # the values observed at `path` on + # each resolved response, in dispatch + # order. Empty when no response + # carried the path. + # cross_response_count_distinct + # → object { observed_cardinality, + # observed_values: [...] } — the + # number of distinct values seen at + # `path` across the resolved response + # set, plus the values themselves. # `actual` is the diagnostic summary. - schema_id # $id of the response schema applied. Required # when check == response_schema, null otherwise. diff --git a/static/compliance/source/universal/storyboard-schema.yaml b/static/compliance/source/universal/storyboard-schema.yaml index 7476ec5329..8824086a57 100644 --- a/static/compliance/source/universal/storyboard-schema.yaml +++ b/static/compliance/source/universal/storyboard-schema.yaml @@ -1438,6 +1438,73 @@ # - "audiences[*].add[*].hashed_email" # - "audiences[*].add[*].external_id" # +# Cross-response assertions — `check: cross_response_field_equal` / `cross_response_count_distinct`: +# These check kinds operate on the RESPONSE SET of a step that dispatches +# multiple parallel requests under the `parallel_dispatch_runner` contract +# (see test-kits/parallel-dispatch-runner.yaml). They are NOT applicable to +# single-dispatch steps: a single response trivially has cardinality 1 on any +# field. Runners MUST gate these checks on the step declaring +# `requires_contract: parallel_dispatch_runner` and a `parallel_dispatch` +# block; outside that gate the check MUST grade `not_applicable`. The +# canonical use case is rule-9 testing (concurrent retries / first-insert-wins +# from L1/security.mdx#idempotency) — fire N parallel calls with the same +# `idempotency_key`, assert exactly one resource was created. +# +# `parallel_dispatch` is a step-level block that travels alongside `requires_contract`: +# +# - id: concurrent_create +# task: create_media_buy +# requires_contract: parallel_dispatch_runner +# parallel_dispatch: +# count: 2 # required, integer 2..10. The runner fires +# # `count` logically-simultaneous requests +# # using the SDK's batch primitive. +# same_idempotency_key: true # required for rule-9 tests; future +# # non-idempotency parallel tests MAY set +# # this false. +# barrier_timeout_ms: 5000 # optional; default 5000 — how long the +# # runner waits at the dispatch barrier +# # before firing requests anyway. +# validations: +# - check: cross_response_count_distinct +# path: "media_buy_id" +# allowed_values: [1] +# description: "Exactly one media_buy_id across all dispatches" +# - check: cross_response_field_equal +# path: "media_buy_id" +# description: "All dispatches converge on the same media_buy_id" +# +# `cross_response_field_equal`: +# - path (string, required): JSON path checked on every resolved response. +# All non-null observed values at `path` MUST be deeply equal. A missing +# value on a response is treated as a non-equal observation and fails the +# check. +# +# `cross_response_count_distinct`: +# - path (string, required): JSON path scanned on every resolved response. +# - allowed_values (array of integers, required): permitted cardinalities of +# distinct values observed at `path` across the resolved response set. +# For rule-9 tests, `allowed_values: [1]` — exactly one resource was +# created across all dispatches. +# +# Resolution semantics. A seller adopting the reject-and-redirect policy under +# rule 9 may return `IDEMPOTENCY_IN_FLIGHT` on the second (or later) dispatch +# rather than block. The runner MUST resolve `IDEMPOTENCY_IN_FLIGHT` responses +# to their eventual cached response before applying these checks: wait +# `error.details.retry_after` seconds and retry with the SAME idempotency_key, +# exactly as the buyer-side normative behavior specifies. The cross-response +# check operates on the resolved response set, not the raw initial set. If +# resolution exceeds the runner's step-timeout budget, the step grades +# `step_timeout` (a runner-synthesized code), not a rule-9 violation. +# +# Grading output. `expected` carries the declared assertion fields +# (`path`, plus `allowed_values` for `count_distinct`). `actual` summarizes +# what was observed across the resolved response set: +# cross_response_field_equal → { observed_values: [...] } +# cross_response_count_distinct → { observed_cardinality, observed_values: [...] } +# See runner-output-contract.yaml > validation_result > expected/actual for +# the canonical shapes. +# # --- Runner output --- # # How a runner MUST report validation results — including which fields a From 4d98a0a198e27ff786e03efb71b1b7dc5b037c74 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 11 May 2026 10:43:01 -0400 Subject: [PATCH 3/3] =?UTF-8?q?spec(idempotency):=20round-3=20expert=20rev?= =?UTF-8?q?iew=20=E2=80=94=20soften=20raw-key=20clause,=20fail-closed=20on?= =?UTF-8?q?=20ambiguous=20downstream=20lookup,=20rename=20lint=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 expert review surfaced 3 single-reviewer must-fix items (no convergent blockers). All three addressed: - Rule 10 raw-key clause was too absolute (security review #5). The prior MUST-NOT-pass-raw forbade legitimate intra-tenant patterns where a seller threads the key into a microservice it operates end-to-end (same KMS, same audit log, same operator). Reworded to forbid passing across trust boundaries specifically — same-trust- principal forwarding is permitted, though per-provider HMAC derivation remains the better default. - Rule 10 claim-row-without-invoke fail-closed on ambiguous lookup (security review #7-low). The prior "if downstream reports no record, proceed with invocation" wording assumed the downstream's lookup response is authentic. Added: on transient 5xx, network error, or malformed response, the seller MUST fail closed (return a transient error to the buyer per rule 9), not proceed on an unauthenticated "no record" signal. - Renamed lint helper loadAuthoredCheckKinds → loadKnownCheckKinds to match its actual behavior (returns the union of authored + cross- response check kinds after PR's round-2 split). Pure naming; lint output and behavior unchanged. Updated tests/lint-storyboard- check-enum.test.cjs to match. Follow-up filed: adcontextprotocol/adcp#4406 — declare capabilities.idempotency.in_flight_max_seconds so buyers can compute retry budgets bounded by the seller's handler-timeout rather than the much-wider replay_ttl_seconds ceiling. Not blocking this PR; additive 3.1 capability declaration. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/building/by-layer/L1/security.mdx | 4 ++-- scripts/lint-storyboard-check-enum.cjs | 6 +++--- tests/lint-storyboard-check-enum.test.cjs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/building/by-layer/L1/security.mdx b/docs/building/by-layer/L1/security.mdx index 661c4c684b..c0d541519d 100644 --- a/docs/building/by-layer/L1/security.mdx +++ b/docs/building/by-layer/L1/security.mdx @@ -320,8 +320,8 @@ This section applies only to AdCP task requests. OpenRTB bid streams have their Sellers MUST adopt one of two reconciliation patterns for every downstream call whose duplicate-invocation has business consequences (resource creation, payment movement, irreversible state change). Read-only downstream calls (cache lookups, eligibility checks that don't write) are exempt — but borderline cases like fraud-scoring lookups that also write to a downstream audit log count as writes for this rule (the audit log entry is the side effect). - - **Write-claim-before-invoke (preferred default).** Before invoking the downstream, the seller persists a "claim" row in the same transaction as the idempotency cache row — typically `{idempotency_key, downstream_provider, downstream_request_id, status: 'invoked', invoked_at}` — using the seller-generated `downstream_request_id` it will pass to the downstream as the downstream's own correlation/idempotency identifier. On retry, before invoking the downstream again, the seller MUST look up the claim row by `(idempotency_key, downstream_provider)` and reconcile: query the downstream by `downstream_request_id` to determine the true outcome, then resume cache population from there. The seller MUST NOT treat a missing local record as "downstream call did not happen" — a crash between downstream-accepts and local-persist is exactly the case where it did happen and the local record is missing. If the downstream reports no record of `downstream_request_id` (the claim row was persisted but the seller crashed before invoking), the seller MUST treat the call as not-yet-invoked and proceed with the invocation; the claim row already reserves the `downstream_request_id`, so the downstream's own idempotency will dedup any subsequent retry. - - **Thread-buyer-key (acceptable when the downstream protocol supports it).** The seller passes a per-downstream-provider derivative of the buyer's `idempotency_key` as the downstream's own idempotency key — typically `HMAC(K_provider, idempotency_key)` where `K_provider` is derived from the seller's KMS-managed root keyed by provider identity (one key per downstream, not one shared seller secret across all downstreams). Per-provider derivation prevents cross-provider replay if any single downstream is compromised; a shared seller secret across all downstreams collapses every provider into a single key-exposure blast radius. The downstream's at-most-once guarantee then covers the case the seller's local persistence missed. The seller MUST still write a claim row on the success path so the cached response can be populated correctly, but the downstream itself becomes the source of truth on retry. The seller MUST NOT pass the buyer's raw `idempotency_key` to a downstream — the buyer's key is a capability token within its TTL (see "Keys are security-sensitive" below) and forwarding it across the seller-downstream trust boundary widens the capability surface. + - **Write-claim-before-invoke (preferred default).** Before invoking the downstream, the seller persists a "claim" row in the same transaction as the idempotency cache row — typically `{idempotency_key, downstream_provider, downstream_request_id, status: 'invoked', invoked_at}` — using the seller-generated `downstream_request_id` it will pass to the downstream as the downstream's own correlation/idempotency identifier. On retry, before invoking the downstream again, the seller MUST look up the claim row by `(idempotency_key, downstream_provider)` and reconcile: query the downstream by `downstream_request_id` to determine the true outcome, then resume cache population from there. The seller MUST NOT treat a missing local record as "downstream call did not happen" — a crash between downstream-accepts and local-persist is exactly the case where it did happen and the local record is missing. If the downstream reports no record of `downstream_request_id` (the claim row was persisted but the seller crashed before invoking), the seller MUST treat the call as not-yet-invoked and proceed with the invocation; the claim row already reserves the `downstream_request_id`, so the downstream's own idempotency will dedup any subsequent retry. On an ambiguous response from the downstream lookup (transient 5xx, network error, malformed response), the seller MUST fail closed — return a transient error to the buyer (so the buyer retries against the same `idempotency_key` per rule 9) rather than proceed with invocation on an unauthenticated "no record" signal. + - **Thread-buyer-key (acceptable when the downstream protocol supports it).** The seller passes a per-downstream-provider derivative of the buyer's `idempotency_key` as the downstream's own idempotency key — typically `HMAC(K_provider, idempotency_key)` where `K_provider` is derived from the seller's KMS-managed root keyed by provider identity (one key per downstream, not one shared seller secret across all downstreams). Per-provider derivation prevents cross-provider replay if any single downstream is compromised; a shared seller secret across all downstreams collapses every provider into a single key-exposure blast radius. The downstream's at-most-once guarantee then covers the case the seller's local persistence missed. The seller MUST still write a claim row on the success path so the cached response can be populated correctly, but the downstream itself becomes the source of truth on retry. The seller MUST NOT pass the buyer's raw `idempotency_key` to any downstream operated by a different trust principal — the buyer's key is a capability token within its TTL (see "Keys are security-sensitive" below) and forwarding it across a trust boundary widens the capability surface. "Different trust principal" means any system the seller does not operate under the same security boundary; passing the raw key to a purely intra-tenant microservice the seller owns end-to-end (same KMS, same audit log, same operator) does NOT cross a trust boundary and is permitted, though per-provider derivation is still the better default. Sellers MUST document which pattern applies to which downstream in their operational runbook. Sellers MUST NOT use a third pattern of "best-effort dedup on downstream response inspection" — comparing the downstream's response payload to a cached fingerprint to decide whether the call already happened — because the downstream's response shape changes across versions and the fingerprint is a synchronization bug waiting to happen. A claim row OR a threaded key. Not pattern-match-on-response. diff --git a/scripts/lint-storyboard-check-enum.cjs b/scripts/lint-storyboard-check-enum.cjs index 6d85aaad9b..fa63b5fcaf 100644 --- a/scripts/lint-storyboard-check-enum.cjs +++ b/scripts/lint-storyboard-check-enum.cjs @@ -49,7 +49,7 @@ const SYNTHESIZED_CHECK_KINDS = new Set([ 'unresolved_substitution', ]); -function loadAuthoredCheckKinds() { +function loadKnownCheckKinds() { const doc = yaml.load(fs.readFileSync(CONTRACT_FILE, 'utf8')); const authored = doc && Array.isArray(doc.authored_check_kinds) ? doc.authored_check_kinds : null; if (!authored || authored.length === 0) { @@ -109,7 +109,7 @@ function* walkValidations(doc) { } function lint(sourceDir = SOURCE_DIR) { - const authoredKinds = loadAuthoredCheckKinds(); + const authoredKinds = loadKnownCheckKinds(); const violations = []; function lintFile(p) { @@ -184,6 +184,6 @@ if (require.main === module) main(); module.exports = { RULE_MESSAGES, SYNTHESIZED_CHECK_KINDS, - loadAuthoredCheckKinds, + loadKnownCheckKinds, lint, }; diff --git a/tests/lint-storyboard-check-enum.test.cjs b/tests/lint-storyboard-check-enum.test.cjs index 9f83d41f00..3adb3db8a7 100644 --- a/tests/lint-storyboard-check-enum.test.cjs +++ b/tests/lint-storyboard-check-enum.test.cjs @@ -20,7 +20,7 @@ const assert = require('node:assert/strict'); const { lint, - loadAuthoredCheckKinds, + loadKnownCheckKinds, SYNTHESIZED_CHECK_KINDS, RULE_MESSAGES, } = require('../scripts/lint-storyboard-check-enum.cjs'); @@ -36,7 +36,7 @@ test('source tree passes the check-enum lint', () => { }); test('authored_check_kinds enum loads from runner-output-contract.yaml', () => { - const kinds = loadAuthoredCheckKinds(); + const kinds = loadKnownCheckKinds(); // Spot-check a few load-bearing entries — keep this tight so the test // doesn't have to track every authored kind. Full enum is the contract. for (const expected of ['response_schema', 'field_present', 'upstream_traffic']) {