diff --git a/.changeset/a2a-context-continuity-validator.md b/.changeset/a2a-context-continuity-validator.md new file mode 100644 index 000000000..12054a85c --- /dev/null +++ b/.changeset/a2a-context-continuity-validator.md @@ -0,0 +1,42 @@ +--- +'@adcp/client': minor +--- + +Storyboard runner: add `a2a_context_continuity` validation check for +multi-step A2A storyboards. Closes the sibling regression class to +adcp-client#952 — sellers that bypass the `@a2a-js/sdk` +`DefaultRequestHandler` and stamp their own `contextId` on the response +Task (instead of echoing the buyer-supplied value) will now fail the +storyboard suite. + +A2A 0.3.0 §7.1 lets buyers pass `params.message.contextId` to bind a +follow-up send to an existing server-side context; the server must echo +the same value on the response Task. The SDK handles this automatically +via `createA2AAdapter`'s `requestContext.contextId` forwarding, so SDK- +based sellers pass silently. Sellers that bypass the SDK are only +detectable on multi-step storyboards — this validator is that gate. + +Runner plumbing: +- `ExecutionState.lastA2aContextId` accumulates the contextId from each + A2A step's response Task. Before each A2A dispatch, the current value + is forwarded as `TaskOptions.contextId` (which rides on + `params.message.contextId` on the wire via `TaskExecutor`). After + dispatch, the response Task's contextId updates the state for the next + step. +- `outboundA2aContextId` is captured before dispatch and passed to + `ValidationContext` so the validator can compare "what was sent" to + "what came back" — avoiding the tautology of asserting against a value + the runner never actually forwarded. +- `executeStoryboardTask` accepts a `contextId` in its opts argument and + threads it through `TaskOptions`, which `SingleAgentClient` and + `TaskExecutor` already forward to `callA2ATool`. + +Validator behavior: +- Self-skips with `not_applicable` when: no `outboundA2aContextId` + (first A2A step, non-A2A run, or prior step had no contextId), + no `a2aEnvelope` (capture miss), or a JSON-RPC error envelope (no + Task returned, so continuity is not verifiable). +- Fails when response `Task.contextId` ≠ `outboundA2aContextId`, or + when response `Task.contextId` is absent/empty on a follow-up send. + +9 unit tests added. Refs adcp-client#962. diff --git a/package-lock.json b/package-lock.json index 5d9521454..34ef0c006 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@adcp/client", - "version": "5.16.0", + "version": "5.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@adcp/client", - "version": "5.16.0", + "version": "5.17.0", "license": "Apache-2.0", "dependencies": { "ajv": "^8.18.0", diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 4079bddee..e5a76ee71 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -1303,6 +1303,15 @@ interface ExecutionState { * the shallow-merge semantics of the context itself. */ contextProvenance?: Map; + /** + * A2A contextId from the most recent step that returned a non-empty + * Task.contextId. Forwarded as `TaskOptions.contextId` on the next + * A2A dispatch so multi-step storyboards bind follow-up sends to the + * same server-side context. `a2a_context_continuity` reads the value + * that was forwarded (i.e. what existed BEFORE a step ran) to confirm + * the seller echoed it back. Issue #962. + */ + lastA2aContextId?: string; } async function executeStep( @@ -1497,6 +1506,7 @@ async function executeStep( let httpResult: HttpProbeResult | undefined; let responseRecord: RunnerResponseRecord | undefined; let a2aEnvelope: A2ATaskEnvelope | undefined; + let outboundA2aContextId: string | undefined; if (useRawProbe) { const started = Date.now(); @@ -1545,10 +1555,17 @@ async function executeStep( // `agentTools` later). If a future "auto-detect protocol" flow // lands, key the capture off the negotiated transport instead. const captureA2a = options.protocol === 'a2a'; + // Snapshot the contextId we are about to forward on this step's outbound + // A2A request. Captured BEFORE dispatch so `a2a_context_continuity` can + // compare it against the response — the runner updates `lastA2aContextId` + // AFTER dispatch (with the response's contextId), so reading it here + // gives us "what was sent", not "what came back". + outboundA2aContextId = captureA2a ? runState.lastA2aContextId : undefined; let a2aCaptures: RawHttpCapture[] | undefined; const dispatch = () => executeStoryboardTask(client, effectiveStep.task, request, { skipIdempotencyAutoInject: testsMissingIdempotencyKey, + ...(outboundA2aContextId && { contextId: outboundA2aContextId }), }); const run = await runStep(step.title, effectiveStep.task, async () => { if (!captureA2a) return dispatch(); @@ -1572,6 +1589,19 @@ async function executeStep( if (captureA2a && a2aCaptures) { a2aEnvelope = parseLastA2aMessageSendCapture(a2aCaptures); } + // Update the session contextId from the response so the NEXT step + // forwards it. Only update when the response Task carried a non-empty + // string — preserves the existing contextId on error paths or when + // the seller omitted the field. + if (captureA2a && a2aEnvelope) { + const responseTask = a2aEnvelope.result; + if (responseTask != null && typeof responseTask === 'object' && !Array.isArray(responseTask)) { + const responseCtxId = (responseTask as Record).contextId; + if (typeof responseCtxId === 'string' && responseCtxId.length > 0) { + runState.lastA2aContextId = responseCtxId; + } + } + } if (taskResult) { responseRecord = { transport: options.protocol === 'a2a' ? 'a2a' : 'mcp', @@ -1658,6 +1688,7 @@ async function executeStep( ...(responseRecord && { response: responseRecord }), storyboardContext: context, ...(a2aEnvelope && { a2aEnvelope }), + ...(outboundA2aContextId && { outboundA2aContextId }), }; validations = runValidations(resolvedValidations, vctx); } diff --git a/src/lib/testing/storyboard/task-map.ts b/src/lib/testing/storyboard/task-map.ts index 0eb780f90..2453b49e1 100644 --- a/src/lib/testing/storyboard/task-map.ts +++ b/src/lib/testing/storyboard/task-map.ts @@ -81,13 +81,19 @@ export async function executeStoryboardTask( client: any, taskName: string, params: Record, - opts: { skipIdempotencyAutoInject?: boolean } = {} + opts: { skipIdempotencyAutoInject?: boolean; contextId?: string } = {} ): Promise { const methodName = Object.hasOwn(TASK_TO_METHOD, taskName) ? TASK_TO_METHOD[taskName] : undefined; // Only pass TaskOptions when a flag is actually set — avoids changing // behavior for the common path that relies on method defaults. - const taskOptions = opts.skipIdempotencyAutoInject ? { skipIdempotencyAutoInject: true } : undefined; + const taskOptions = + opts.skipIdempotencyAutoInject || opts.contextId + ? { + ...(opts.skipIdempotencyAutoInject && { skipIdempotencyAutoInject: true }), + ...(opts.contextId && { contextId: opts.contextId }), + } + : undefined; let result; const invoke = async () => { diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index 2c076e842..8d3e20885 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -402,6 +402,7 @@ export type StoryboardValidationCheck = | 'any_of' // A2A wire-shape checks (transport-specific; skipped on non-A2A runs) | 'a2a_submitted_artifact' + | 'a2a_context_continuity' // Cross-step checks | 'refs_resolve'; diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index f33416ca1..559f13085 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -63,6 +63,19 @@ export interface ValidationContext { * those checks self-skip with `not_applicable`. */ a2aEnvelope?: A2ATaskEnvelope; + /** + * The `contextId` that was forwarded on the outbound A2A `message/send` + * for this step — i.e., the value the runner passed as + * `TaskOptions.contextId`, which rode on `params.message.contextId` on + * the wire. Populated only when a prior A2A step returned a contextId + * that the runner carried forward. `a2a_context_continuity` uses this + * to confirm the seller echoed the buyer-supplied id rather than + * stamping a new one. + * + * Absent on the first A2A step (no prior contextId to forward), on MCP + * runs, and on runs where the prior step's envelope carried no contextId. + */ + outboundA2aContextId?: string; } /** @@ -114,6 +127,8 @@ function runValidation(validation: StoryboardValidation, ctx: ValidationContext) return validateAnyOf(validation, ctx.contributions); case 'a2a_submitted_artifact': return validateA2ASubmittedArtifact(validation, ctx); + case 'a2a_context_continuity': + return validateA2AContextContinuity(validation, ctx); case 'refs_resolve': return validateRefsResolve(validation, ctx); default: @@ -1528,6 +1543,124 @@ function validateA2ASubmittedArtifact(validation: StoryboardValidation, ctx: Val }; } +// ──────────────────────────────────────────────────────────── +// a2a_context_continuity (multi-step A2A session guard) +// ──────────────────────────────────────────────────────────── + +/** + * Assert that the seller echoes the buyer-supplied `contextId` on every + * follow-up `message/send` — confirming A2A session continuity per + * A2A 0.3.0 §7.1. + * + * A2A 0.3.0 §7.1 lets buyers bind follow-up sends to an existing server- + * side context by setting `params.message.contextId`; the server MUST + * echo the same value on the response Task. The `@a2a-js/sdk` + * `DefaultRequestHandler` does this automatically (via + * `createA2AAdapter`'s `requestContext.contextId` forwarding), so SDK- + * based sellers pass transparently. Sellers that bypass the SDK and stamp + * their own contextId only fail on multi-step storyboards — this check + * is that multi-step gate. + * + * The check runs at step N+1: the runner captured contextId from step N's + * response, forwarded it as `TaskOptions.contextId` on the step N+1 + * dispatch (which rides on `params.message.contextId` on the wire), and + * now verifies the response Task echoes the same value. + * + * Self-skips with `not_applicable` when: + * - No `outboundA2aContextId` — first A2A step, non-A2A run, or prior + * step had no contextId to forward. + * - No `a2aEnvelope` — capture didn't fire (raw probe path, SDK error). + * + * Hard-fails when: + * - JSON-RPC error envelope (can't verify contextId; distinct error_code + * so dashboards separate transport errors from continuity breaks). + * - Response Task.contextId ≠ outbound contextId (seller stamped new id). + * - Response Task.contextId absent/empty on follow-up (continuity break). + */ +function validateA2AContextContinuity(validation: StoryboardValidation, ctx: ValidationContext): ValidationResult { + // Skip when there is no prior contextId to forward (first step or non-A2A). + if (!ctx.outboundA2aContextId) { + return { + check: 'a2a_context_continuity', + passed: true, + description: validation.description, + observations: [ + 'no_prior_a2a_context_id: no contextId was forwarded on this send (first A2A step, non-A2A run, or prior step carried no contextId)', + ], + }; + } + + const envelope = ctx.a2aEnvelope; + if (!envelope) { + return { + check: 'a2a_context_continuity', + passed: true, + description: validation.description, + observations: [ + 'a2a_envelope_not_captured: no JSON-RPC envelope recorded (non-A2A transport, or A2A dispatch threw before envelope was parsed)', + ], + }; + } + + // JSON-RPC error envelopes carry no Task, so there is no contextId to + // check. Skip with not_applicable rather than hard-failing — the seller + // may have rejected the request for reasons entirely unrelated to context + // binding (auth, rate-limit, schema error, business logic), and flagging + // that as a "continuity break" would produce false positives. Contrast + // with a2a_submitted_artifact, which hard-fails error envelopes because + // the submitted-arm shape check is specifically about the Task payload; + // here the invariant is only meaningful when a Task was returned. + if (envelope.envelope.error !== undefined) { + return { + check: 'a2a_context_continuity', + passed: true, + description: validation.description, + observations: [ + `a2a_jsonrpc_error_envelope: server returned a JSON-RPC error; contextId continuity check skipped (no Task to verify)`, + ], + }; + } + + const result = envelope.result; + if (result == null || typeof result !== 'object' || Array.isArray(result)) { + return { + check: 'a2a_context_continuity', + passed: false, + description: validation.description, + error: 'JSON-RPC `result` is not an object — A2A `message/send` must return a Task.', + json_pointer: '/result', + expected: 'object (A2A Task)', + actual: result, + }; + } + const task = result as Record; + const responseContextId = + typeof task.contextId === 'string' && task.contextId.length > 0 ? task.contextId : undefined; + const outbound = ctx.outboundA2aContextId; + + if (responseContextId === outbound) { + return { + check: 'a2a_context_continuity', + passed: true, + description: validation.description, + }; + } + + const detail = responseContextId + ? `A2A Task.contextId '${responseContextId}' does not match the forwarded contextId '${outbound}' — seller must echo the buyer-supplied contextId per A2A 0.3.0 §7.1.` + : `A2A Task.contextId was absent or empty on a follow-up send — seller must echo the buyer-supplied contextId '${outbound}' per A2A 0.3.0 §7.1.`; + + return { + check: 'a2a_context_continuity', + passed: false, + description: validation.description, + error: detail, + json_pointer: '/result/contextId', + expected: outbound, + actual: task.contextId ?? null, + }; +} + // ──────────────────────────────────────────────────────────── // refs_resolve (cross-step integrity check) // ──────────────────────────────────────────────────────────── diff --git a/test/storyboard-a2a-context-continuity.test.js b/test/storyboard-a2a-context-continuity.test.js new file mode 100644 index 000000000..ef64a1099 --- /dev/null +++ b/test/storyboard-a2a-context-continuity.test.js @@ -0,0 +1,132 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert'); + +const { runValidations } = require('../dist/lib/testing/storyboard/validations.js'); + +const VALIDATION = { + check: 'a2a_context_continuity', + description: 'Seller echoes buyer-supplied contextId on follow-up A2A send', +}; + +const PRIOR_CTX_ID = 'ctx-prior-step-uuid'; + +function ctx({ envelope, outboundA2aContextId } = {}) { + return { + taskName: 'create_media_buy', + agentUrl: 'https://example.com/a2a', + contributions: new Set(), + ...(envelope !== undefined && { a2aEnvelope: envelope }), + ...(outboundA2aContextId !== undefined && { outboundA2aContextId }), + }; +} + +function makeEnvelope({ contextId = PRIOR_CTX_ID } = {}) { + return { + result: { + kind: 'task', + id: 'a2a-task-uuid', + contextId, + status: { state: 'completed', timestamp: '2026-04-25T00:00:00Z' }, + artifacts: [], + }, + envelope: { jsonrpc: '2.0', id: 1, result: {} }, + http_status: 200, + }; +} + +describe('a2a_context_continuity', () => { + it('passes when seller echoes the forwarded contextId', () => { + const [result] = runValidations( + [VALIDATION], + ctx({ envelope: makeEnvelope({ contextId: PRIOR_CTX_ID }), outboundA2aContextId: PRIOR_CTX_ID }) + ); + assert.strictEqual(result.passed, true); + assert.strictEqual(result.check, 'a2a_context_continuity'); + assert.ok(!result.error); + }); + + it('passes with not_applicable when no outboundA2aContextId (first A2A step)', () => { + const [result] = runValidations([VALIDATION], ctx({ envelope: makeEnvelope(), outboundA2aContextId: undefined })); + assert.strictEqual(result.passed, true); + assert.ok(Array.isArray(result.observations)); + assert.ok(result.observations[0].includes('no_prior_a2a_context_id')); + }); + + it('passes with not_applicable when outboundA2aContextId is absent (non-A2A transport)', () => { + // No envelope and no outbound id — non-A2A MCP run + const [result] = runValidations([VALIDATION], ctx({})); + assert.strictEqual(result.passed, true); + assert.ok(Array.isArray(result.observations)); + assert.ok(result.observations[0].includes('no_prior_a2a_context_id')); + }); + + it('passes with not_applicable when envelope absent but outboundA2aContextId set (capture miss)', () => { + const [result] = runValidations([VALIDATION], ctx({ envelope: undefined, outboundA2aContextId: PRIOR_CTX_ID })); + assert.strictEqual(result.passed, true); + assert.ok(Array.isArray(result.observations)); + assert.ok(result.observations[0].includes('a2a_envelope_not_captured')); + }); + + it('fails when response Task.contextId differs from outbound contextId (seller stamped new id)', () => { + const [result] = runValidations( + [VALIDATION], + ctx({ + envelope: makeEnvelope({ contextId: 'ctx-SELLER-STAMPED-WRONG' }), + outboundA2aContextId: PRIOR_CTX_ID, + }) + ); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.check, 'a2a_context_continuity'); + assert.strictEqual(result.json_pointer, '/result/contextId'); + assert.strictEqual(result.expected, PRIOR_CTX_ID); + assert.strictEqual(result.actual, 'ctx-SELLER-STAMPED-WRONG'); + assert.match(result.error, /does not match the forwarded contextId/); + }); + + it('fails when response Task.contextId is absent (continuity break — seller dropped it)', () => { + const env = makeEnvelope(); + delete env.result.contextId; + const [result] = runValidations([VALIDATION], ctx({ envelope: env, outboundA2aContextId: PRIOR_CTX_ID })); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.json_pointer, '/result/contextId'); + assert.strictEqual(result.expected, PRIOR_CTX_ID); + assert.strictEqual(result.actual, null); + assert.match(result.error, /absent or empty on a follow-up send/); + }); + + it('fails when response Task.contextId is empty string (treated as absent)', () => { + const [result] = runValidations( + [VALIDATION], + ctx({ + envelope: makeEnvelope({ contextId: '' }), + outboundA2aContextId: PRIOR_CTX_ID, + }) + ); + assert.strictEqual(result.passed, false); + assert.match(result.error, /absent or empty on a follow-up send/); + }); + + it('passes with not_applicable when JSON-RPC error envelope (no Task to verify contextId on)', () => { + const env = { + result: null, + envelope: { jsonrpc: '2.0', id: 1, error: { code: -32600, message: 'Invalid Request' } }, + http_status: 200, + }; + const [result] = runValidations([VALIDATION], ctx({ envelope: env, outboundA2aContextId: PRIOR_CTX_ID })); + assert.strictEqual(result.passed, true); + assert.ok(Array.isArray(result.observations)); + assert.ok(result.observations[0].includes('a2a_jsonrpc_error_envelope')); + }); + + it('fails when result is not an object', () => { + const env = { + result: 'not-an-object', + envelope: { jsonrpc: '2.0', id: 1, result: 'not-an-object' }, + http_status: 200, + }; + const [result] = runValidations([VALIDATION], ctx({ envelope: env, outboundA2aContextId: PRIOR_CTX_ID })); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.json_pointer, '/result'); + assert.match(result.error, /not an object/); + }); +});