From 7ae37c30eb45c12f25919cefb2921474dc554b36 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sun, 19 Apr 2026 07:26:24 -0400 Subject: [PATCH] feat: conform runner output to universal runner-output contract (#599) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #599. The storyboard runner and comply() output now emit the minimum failure detail required for implementors to self-diagnose: RFC 6901 JSON Pointer to the failing field, machine-readable expected / actual, the schema $id that was applied (plus a resolvable URL), the MCP extraction path, and the exact request/response the runner sent. Previously a failed storyboard step surfaced as "Check agent capabilities" with no actionable detail — reporters in adcontextprotocol/adcp#2352 exhausted every locally-verifiable hypothesis before filing. The runner now answers "which field", "against which schema", and "was this a structured-content extraction or a text-fallback" in the structured result, and the plain-text comply() formatter surfaces the pointer, schema URL, and extraction path in its "How to Fix" section so reports are actionable without --json. ValidationResult gains json_pointer / expected / actual / schema_id / schema_url. StoryboardStepResult gains request_record / response_record / extraction / skip_detail. Skip reasons map to the contract enum (not_applicable, no_phases, prerequisite_failed, missing_tool, missing_test_controller, unsatisfied_contract) with detail strings citing declared tools / missing tool / prerequisite id. The legacy skip reason values remain in the type union so existing consumers don't break. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../runner-output-contract-conformance.md | 29 +++ src/lib/testing/compliance/comply.ts | 34 ++++ src/lib/testing/compliance/types.ts | 17 ++ src/lib/testing/orchestrator.ts | 44 +++- src/lib/testing/storyboard/path.ts | 13 ++ src/lib/testing/storyboard/runner.ts | 160 ++++++++++++++- src/lib/testing/storyboard/types.ts | 98 ++++++++- src/lib/testing/storyboard/validations.ts | 104 ++++++++-- src/lib/testing/types.ts | 19 ++ src/lib/utils/response-schemas.ts | 115 +++++++++++ test/lib/runner-output-contract.test.js | 191 ++++++++++++++++++ 11 files changed, 793 insertions(+), 31 deletions(-) create mode 100644 .changeset/runner-output-contract-conformance.md create mode 100644 test/lib/runner-output-contract.test.js diff --git a/.changeset/runner-output-contract-conformance.md b/.changeset/runner-output-contract-conformance.md new file mode 100644 index 000000000..44570efbb --- /dev/null +++ b/.changeset/runner-output-contract-conformance.md @@ -0,0 +1,29 @@ +--- +'@adcp/client': minor +--- + +Conform storyboard runner and `comply()` output to the universal runner-output contract (#599, adcontextprotocol/adcp#2364). + +A failing storyboard validation is only actionable when the report names the offending field, the expected value, what the runner observed, and which schema artifact to re-validate against locally. Previously the runner emitted `FAILED: schema_validation/capability_discovery — "Check agent capabilities"` with no structured detail — an implementor whose local AJV passed couldn't tell whether the issue was a missing field, a schema mismatch, a transport extraction bug, or a genuine agent bug. This release closes that gap. + +**`ValidationResult` now carries machine-readable failure detail.** Every validation result populates the contract's required-on-failure fields: + +- `json_pointer` — RFC 6901 pointer to the failing field (`/adcp/idempotency` instead of `adcp.idempotency`). +- `expected` — the schema `$id` for `response_schema`, the expected value / allowed_values for field checks, the expected status for HTTP checks, etc. +- `actual` — the observed value; for `response_schema` an AJV-style array of `{ instance_path, keyword, message }` so consumers can pinpoint every violation, not just the first. +- `schema_id` — the `$id` applied when `check === 'response_schema'`, e.g. `/schemas/latest/protocol/get-adcp-capabilities-response.json`. +- `schema_url` — resolvable URL for local re-validation, e.g. `https://adcontextprotocol.org/schemas/latest/protocol/get-adcp-capabilities-response.json`. + +The existing `error` string is preserved as a human-readable fallback so current consumers continue to render a one-line message. + +**`StoryboardStepResult` now carries the exact request, response, and extraction path.** New optional fields: + +- `request_record` — transport + operation + fully-resolved payload the runner sent (secrets redacted). +- `response_record` — transport + observed payload + HTTP status/headers when applicable. +- `extraction` — `{ path: 'structured_content' | 'text_fallback' | 'error' | 'none' }` so an implementor can distinguish a runner extraction bug from an agent bug per `docs/building/implementation/mcp-response-extraction`. `path` is classified from the task result; `text_fallback` fires when the response unwrapper synthesized an error from non-JSON text content. + +**Skip reasons now map to the contract enum with detail strings.** Step results use the contract's skip reasons — `not_applicable`, `no_phases`, `prerequisite_failed`, `missing_tool`, `missing_test_controller`, `unsatisfied_contract` — each carrying a `skip_detail` string that cites the advertised tools, missing tool, prerequisite step id, or contract key. The legacy reason values (`not_testable`, `dependency_failed`, `missing_test_harness`) remain in the type union so existing consumers don't break; the runner now emits the contract names (`dependency_failed` → `prerequisite_failed`, `missing_test_harness` → `missing_test_controller`, `not_testable` → `missing_tool`). + +**`ComplianceFailure` surfaces the contract fields.** Each failure in `ComplianceResult.failures` now includes the failing validations with their `json_pointer` / `expected` / `actual` / `schema_id` / `schema_url`, the step's `extraction` path, and the recorded `request` / `response`. The plain-text `comply()` formatter also surfaces the failing field pointer, schema URL, and extraction path in its "How to Fix" section so reports are actionable without `--json`. + +**Orchestrator**: `getScenarioSkips()` is a new helper that returns contract-shaped skip records (`reason` + `detail`) for scenarios not applicable to the agent's tool list. `testAllScenarios()` populates `SuiteResult.scenarios_skipped_detail`, and the markdown formatter renders each skip with its reason + detail so reports distinguish "scenario not applicable" from "agent claims the protocol but lacks a required tool." diff --git a/src/lib/testing/compliance/comply.ts b/src/lib/testing/compliance/comply.ts index 2b0337c8f..dbc162b0b 100644 --- a/src/lib/testing/compliance/comply.ts +++ b/src/lib/testing/compliance/comply.ts @@ -685,6 +685,7 @@ function extractFailures( } } + const failedValidations = step.validations.filter(v => !v.passed); failures.push({ track, storyboard_id: result.storyboard_id, @@ -694,6 +695,27 @@ function extractFailures( error: step.error, expected, fix_command: `adcp storyboard step ${agentRef} ${result.storyboard_id} ${step.step_id} --json`, + ...(failedValidations.length > 0 && { + validations: failedValidations.map(v => ({ + check: v.check, + description: v.description, + ...(v.json_pointer !== undefined && { json_pointer: v.json_pointer }), + ...(v.expected !== undefined && { expected: v.expected }), + ...(v.actual !== undefined && { actual: v.actual }), + ...(v.schema_id && { schema_id: v.schema_id }), + ...(v.schema_url && { schema_url: v.schema_url }), + ...(v.error && { error: v.error }), + })), + }), + ...(step.extraction && { extraction: step.extraction }), + ...(step.request_record && { request: step.request_record }), + ...(step.response_record && { + response: { + transport: step.response_record.transport, + payload: step.response_record.payload, + ...(step.response_record.status !== undefined && { status: step.response_record.status }), + }, + }), }); } } @@ -1266,6 +1288,18 @@ export function formatComplianceResults(result: ComplianceResult): string { output += `❌ ${f.storyboard_id}/${f.step_id} (${f.task})\n`; if (f.error) output += ` Error: ${f.error}\n`; output += ` Expected: ${f.expected!.split('\n')[0]}\n`; + // Surface runner-output-contract detail so reports name the offending + // field and schema without forcing consumers into --json. + const firstValidation = f.validations?.[0]; + if (firstValidation?.json_pointer) { + output += ` Field: ${firstValidation.json_pointer}\n`; + } + if (firstValidation?.schema_url) { + output += ` Schema: ${firstValidation.schema_url}\n`; + } + if (f.extraction) { + output += ` Extraction: ${f.extraction.path}${f.extraction.note ? ` (${f.extraction.note})` : ''}\n`; + } output += ` Debug: ${f.fix_command}\n`; } if (failuresWithExpected.length > 5) { diff --git a/src/lib/testing/compliance/types.ts b/src/lib/testing/compliance/types.ts index 7ade20611..7a5da3601 100644 --- a/src/lib/testing/compliance/types.ts +++ b/src/lib/testing/compliance/types.ts @@ -64,6 +64,23 @@ export interface ComplianceFailure { expected?: string; /** CLI command to re-run just this step for debugging */ fix_command: string; + /** Validation failures with runner-output-contract detail (json_pointer, expected, actual, schema_id/url). */ + validations?: Array<{ + check: string; + description: string; + json_pointer?: string; + expected?: unknown; + actual?: unknown; + schema_id?: string; + schema_url?: string; + error?: string; + }>; + /** MCP/A2A extraction path the runner used — separates extraction bugs from agent bugs. */ + extraction?: { path: 'structured_content' | 'text_fallback' | 'error' | 'none'; note?: string }; + /** Exact request the runner sent (secrets redacted). */ + request?: { transport: 'mcp' | 'a2a' | 'http'; operation: string; payload: unknown; url?: string }; + /** Exact response observed. */ + response?: { transport: 'mcp' | 'a2a' | 'http'; payload: unknown; status?: number }; } export interface ComplianceResult { diff --git a/src/lib/testing/orchestrator.ts b/src/lib/testing/orchestrator.ts index 17e06225f..92975816b 100644 --- a/src/lib/testing/orchestrator.ts +++ b/src/lib/testing/orchestrator.ts @@ -7,7 +7,7 @@ import { testAgent } from './agent-tester'; import { createTestClient, discoverAgentProfile, getOrCreateClient, getOrDiscoverProfile } from './client'; -import type { TestScenario, TestOptions, TestResult, SuiteResult } from './types'; +import type { TestScenario, TestOptions, TestResult, SuiteResult, ScenarioSkip } from './types'; /** * Minimum tools required for each scenario to be applicable. @@ -178,6 +178,36 @@ export function getApplicableScenarios(tools: string[], filter?: readonly TestSc return (candidates as TestScenario[]).filter(s => isApplicable(s, tools)); } +/** + * Produce a `ScenarioSkip` record for each scenario not applicable for an + * agent with the given tool list. Emits the runner-output contract's + * `reason` / `detail` distinction so consumers know whether a skip is + * informative (agent didn't claim the protocol — `not_applicable`) or + * actionable (agent claimed it but lacks a required tool — `missing_tool` + * or `missing_test_controller`). + */ +export function getScenarioSkips(tools: string[], filter?: readonly TestScenario[]): ScenarioSkip[] { + const candidates = filter ?? DEFAULT_SCENARIOS; + const skips: ScenarioSkip[] = []; + for (const scenario of candidates) { + if (isApplicable(scenario, tools)) continue; + const requirements = SCENARIO_REQUIREMENTS[scenario] ?? []; + const missing = requirements.filter(t => !tools.includes(t)); + const detail = + missing.length === 0 + ? `Scenario "${scenario}" is not registered.` + : `Agent advertises: ${tools.join(', ') || '(none)'}. Missing: ${missing.join(', ')}.`; + + const reason: ScenarioSkip['reason'] = missing.includes('comply_test_controller') + ? 'missing_test_controller' + : missing.length === 0 + ? 'not_applicable' + : 'missing_tool'; + skips.push({ scenario, reason, detail }); + } + return skips; +} + /** * Run all applicable test scenarios against an agent and return aggregated results. * @@ -229,6 +259,7 @@ export async function testAllScenarios(agentUrl: string, options: OrchestratorOp const applicable = getApplicableScenarios(profile.tools, scenarioFilter); const candidates: readonly TestScenario[] = scenarioFilter ?? DEFAULT_SCENARIOS; const skipped = (candidates as TestScenario[]).filter(s => !applicable.includes(s)); + const skippedDetail = getScenarioSkips(profile.tools, scenarioFilter); // Run each applicable scenario sequentially const results: TestResult[] = []; @@ -245,6 +276,7 @@ export async function testAllScenarios(agentUrl: string, options: OrchestratorOp agent_profile: profile, scenarios_run: applicable, scenarios_skipped: skipped, + scenarios_skipped_detail: skippedDetail, results, overall_passed: failedCount === 0 && passedCount > 0, passed_count: passedCount, @@ -270,7 +302,15 @@ export function formatSuiteResults(suite: SuiteResult): string { } if (suite.scenarios_skipped.length > 0) { - output += `**Skipped (agent does not advertise required tools):** ${suite.scenarios_skipped.join(', ')}\n\n`; + output += `**Skipped:**\n`; + const detailByScenario = new Map((suite.scenarios_skipped_detail ?? []).map(d => [d.scenario, d])); + for (const s of suite.scenarios_skipped) { + const d = detailByScenario.get(s); + output += d + ? `- \`${s}\` — ${d.reason}: ${d.detail}\n` + : `- \`${s}\` — missing_tool\n`; + } + output += '\n'; } output += `### Scenario Results\n\n`; diff --git a/src/lib/testing/storyboard/path.ts b/src/lib/testing/storyboard/path.ts index e6ae9138e..caf7c1289 100644 --- a/src/lib/testing/storyboard/path.ts +++ b/src/lib/testing/storyboard/path.ts @@ -5,6 +5,19 @@ * into segment arrays, and resolves/sets values at those paths. */ +/** + * Convert a dot-path with array indexing to an RFC 6901 JSON Pointer. + * + * "adcp.idempotency" → "/adcp/idempotency" + * "accounts[0].account_id" → "/accounts/0/account_id" + * "a/b~c" → "/a~1b~0c" (escaped per RFC 6901) + */ +export function toJsonPointer(path: string): string { + const segments = parsePath(path); + if (segments.length === 0) return ''; + return '/' + segments.map(s => String(s).replace(/~/g, '~0').replace(/\//g, '~1')).join('/'); +} + /** * Parse a path string into segments. * "accounts[0].account_id" → ["accounts", 0, "account_id"] diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 25dfa01d0..72e99feb0 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -24,7 +24,10 @@ import { } from './probes'; import { validateTestKit } from './test-kit'; import type { + ExtractionRecord, HttpProbeResult, + RecordedRequest, + RecordedResponse, StepAuthDirective, Storyboard, StoryboardStep, @@ -123,6 +126,7 @@ export async function runStoryboard( for (const step of phase.steps) { // Skip remaining steps if a stateful dependency failed if (statefulFailed && step.stateful) { + const priorFailedId = findLastFailedStatefulId(stepResults); stepResults.push({ step_id: step.id, phase_id: phase.id, @@ -130,7 +134,10 @@ export async function runStoryboard( task: step.task, passed: false, skipped: true, - skip_reason: 'dependency_failed', + skip_reason: 'prerequisite_failed', + skip_detail: priorFailedId + ? `Prerequisite step "${priorFailedId}" did not pass.` + : 'A prior stateful step did not pass.', duration_ms: 0, validations: [], context, @@ -337,6 +344,7 @@ async function executeStep( // Check requires_tool — skip if agent doesn't have it if (step.requires_tool && options.agentTools && !options.agentTools.includes(step.requires_tool)) { const next = getNextStepPreview(step.id, allSteps, context); + const isController = step.requires_tool === 'comply_test_controller'; return { step_id: step.id, phase_id: phaseId, @@ -344,7 +352,10 @@ async function executeStep( task: step.task, passed: true, skipped: true, - skip_reason: step.requires_tool === 'comply_test_controller' ? 'missing_test_harness' : 'not_testable', + skip_reason: isController ? 'missing_test_controller' : 'missing_tool', + skip_detail: isController + ? `Deterministic-testing step requires comply_test_controller; agent advertises: ${options.agentTools.join(', ') || '(none)'}.` + : `Step requires tool "${step.requires_tool}"; agent advertises: ${options.agentTools.join(', ') || '(none)'}.`, duration_ms: 0, validations: [], context, @@ -363,6 +374,7 @@ async function executeStep( passed: true, skipped: true, skip_reason: 'missing_tool', + skip_detail: `Agent does not advertise "${effectiveStep.task}". Advertised tools: ${options.agentTools.join(', ') || '(none)'}.`, duration_ms: 0, validations: [], context, @@ -434,7 +446,8 @@ async function executeStep( task: step.task, passed: false, skipped: true, - skip_reason: 'dependency_failed', + skip_reason: 'prerequisite_failed', + skip_detail: `Unresolved context variables from prior steps: ${unresolvedVars.join(', ')}. A producing step did not run or failed.`, duration_ms: 0, validations: [], context, @@ -510,7 +523,10 @@ async function executeStep( task: step.task, passed: true, skipped: true, - skip_reason: isUnknownTool ? 'missing_tool' : 'not_testable', + skip_reason: 'missing_tool', + skip_detail: isUnknownTool + ? `Agent rejected "${effectiveStep.task}" as an unknown tool.` + : `Agent reported the feature is not supported for "${effectiveStep.task}".`, duration_ms: stepResult.duration_ms, validations: [], context, @@ -583,6 +599,15 @@ async function executeStep( // Build next step preview const next = getNextStepPreview(step.id, allSteps, updatedContext); + const extraction = classifyExtraction(taskResult, httpResult); + const request_record = buildRequestRecord( + effectiveStep.task, + request, + step.auth !== undefined ? 'http' : 'mcp', + step.auth !== undefined ? runState.agentUrl : undefined + ); + const response_record = buildResponseRecord(taskResult, httpResult, step.auth !== undefined ? 'http' : 'mcp'); + return { step_id: step.id, phase_id: phaseId, @@ -595,6 +620,9 @@ async function executeStep( validations, context: updatedContext, error: step.expect_error ? undefined : truncateError(stepResult.error || taskResult?.error), + request_record, + response_record, + extraction, next, }; } @@ -653,6 +681,21 @@ async function executeProbeStep( validations, context, error: httpResult?.error ?? (passed ? undefined : 'Probe validations failed.'), + ...(httpResult && { + request_record: { + transport: 'http' as const, + operation: step.task, + payload: null, + url: httpResult.url, + }, + response_record: { + transport: 'http' as const, + payload: httpResult.body, + status: httpResult.status, + headers: httpResult.headers, + }, + }), + extraction: classifyExtraction(undefined, httpResult), next: getNextStepPreview(step.id, allSteps, context), }; } @@ -1023,6 +1066,115 @@ function annotateMultiInstanceFailure( result.error = `${base}\n\n${lines.join('\n')}`; } +// ──────────────────────────────────────────────────────────── +// Runner-output-contract helpers +// ──────────────────────────────────────────────────────────── + +/** + * Best-effort extraction-path classification per the runner-output contract. + * + * Knowable from the storyboard layer: + * - `error` — transport/agent returned an error (synthetic or otherwise) + * - `none` — no data and no error (e.g. probe with no response) + * - `structured_content` — default for successful MCP responses; the response + * unwrapper prefers structuredContent and falls back + * to parsing text-as-JSON. Agents that serve text + * without JSON are surfaced via a synthetic error. + * + * Distinguishing L3 (structuredContent) from L2 (text-parsed-as-JSON) requires + * a side-channel from the SDK that doesn't exist today; both land in the same + * "structured_content" bucket here and the `note` field calls it out. + */ +function classifyExtraction( + taskResult: TaskResult | undefined, + httpResult: HttpProbeResult | undefined +): ExtractionRecord { + if (httpResult) { + if (httpResult.error) return { path: 'error', note: httpResult.error }; + if (httpResult.body === undefined || httpResult.body === null) return { path: 'none' }; + return { path: 'structured_content', note: 'HTTP probe body' }; + } + if (!taskResult) return { path: 'none' }; + const data = taskResult.data as Record | undefined; + const adcpError = data?.adcp_error as Record | undefined; + if (adcpError?.synthetic === true) { + return { path: 'text_fallback', note: 'synthesized from non-JSON text content' }; + } + if (!taskResult.success || taskResult.error) { + return { path: 'error' }; + } + if (data === undefined || data === null) return { path: 'none' }; + return { path: 'structured_content' }; +} + +/** Redact header values that commonly carry secrets. Values themselves may still leak. */ +const REDACTED = '[redacted]'; +const SECRET_HEADER_KEYS = new Set(['authorization', 'proxy-authorization', 'cookie', 'set-cookie']); + +function redactHeaders(headers: Record | undefined): Record | undefined { + if (!headers) return undefined; + const out: Record = {}; + for (const [k, v] of Object.entries(headers)) { + out[k] = SECRET_HEADER_KEYS.has(k.toLowerCase()) ? REDACTED : v; + } + return out; +} + +/** Redact payload fields that carry secrets. Shallow; agents that nest are their own problem. */ +const SECRET_PAYLOAD_KEYS = new Set(['authorization', 'api_key', 'apikey', 'token', 'password', 'secret']); + +function redactPayload(payload: unknown): unknown { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return payload; + const input = payload as Record; + const output: Record = {}; + for (const [k, v] of Object.entries(input)) { + output[k] = SECRET_PAYLOAD_KEYS.has(k.toLowerCase()) && typeof v === 'string' ? REDACTED : v; + } + return output; +} + +function buildRequestRecord( + taskName: string, + payload: Record, + transport: 'mcp' | 'a2a' | 'http', + url?: string +): RecordedRequest { + return { + transport, + operation: taskName, + payload: redactPayload(payload), + ...(url && { url }), + }; +} + +function buildResponseRecord( + taskResult: TaskResult | undefined, + httpResult: HttpProbeResult | undefined, + transport: 'mcp' | 'a2a' | 'http' +): RecordedResponse | undefined { + if (httpResult) { + return { + transport: 'http', + payload: httpResult.body, + status: httpResult.status, + headers: redactHeaders(httpResult.headers), + }; + } + if (!taskResult) return undefined; + return { + transport, + payload: taskResult.data ?? (taskResult.error ? { error: taskResult.error } : null), + }; +} + +function findLastFailedStatefulId(results: StoryboardStepResult[]): string | undefined { + for (let i = results.length - 1; i >= 0; i--) { + const r = results[i]!; + if (!r.skipped && !r.passed) return r.step_id; + } + return undefined; +} + /** * Get a preview of the first step in a storyboard (for showing what will happen). */ diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index d61753357..8c9717778 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -232,14 +232,89 @@ export interface StoryboardRunOptions extends TestOptions { // Results // ──────────────────────────────────────────────────────────── +/** + * Machine-readable detail emitted per the runner-output contract + * (universal/runner-output-contract.yaml). Populated on failures so an + * implementor can self-diagnose without re-running the call by hand. + */ export interface ValidationResult { check: string; passed: boolean; description: string; path?: string; + /** Human-readable fallback message. Structured fields below are authoritative. */ error?: string; + /** RFC 6901 JSON Pointer to the failing field. */ + json_pointer?: string; + /** Machine-readable expected value (or schema $id for response_schema). */ + expected?: unknown; + /** Machine-readable observed value (or AJV-style error array for response_schema). */ + actual?: unknown; + /** $id of the response schema applied. Set when check === "response_schema". */ + schema_id?: string; + /** Resolvable URL for the response schema. Set when check === "response_schema". */ + schema_url?: string; + /** Runner-suggested remediation for known failure signatures. */ + remediation?: string; +} + +/** + * Request the runner sent on the wire, per the runner-output contract. + * `payload` is the fully-resolved body after test-kit substitution and + * context injection. Secrets are redacted. + */ +export interface RecordedRequest { + transport: 'mcp' | 'a2a' | 'http'; + operation: string; + payload: unknown; + headers?: Record; + url?: string; +} + +/** + * Response the runner observed from the agent, per the runner-output contract. + */ +export interface RecordedResponse { + transport: 'mcp' | 'a2a' | 'http'; + payload: unknown; + status?: number; + headers?: Record; + duration_ms?: number; } +/** + * Transport-extraction record per the runner-output contract. Lets an + * implementor separate runner extraction bugs from agent bugs. + */ +export interface ExtractionRecord { + path: 'structured_content' | 'text_fallback' | 'error' | 'none'; + note?: string; +} + +/** + * Contract skip reasons from universal/runner-output-contract.yaml. + * + * - `not_applicable`: agent did not declare the protocol/specialism. + * - `no_phases`: storyboard is a placeholder (no phases to run). + * - `prerequisite_failed`: prior dependency step did not pass. + * - `missing_tool`: agent is missing a tool the step requires. + * - `missing_test_controller`: deterministic-testing phase requires + * `comply_test_controller` and the agent did not advertise it. + * - `unsatisfied_contract`: harness-contract key (e.g., signed-requests-runner) + * permits SKIP for this specialism. + */ +export type StoryboardSkipReason = + | 'not_applicable' + | 'no_phases' + | 'prerequisite_failed' + | 'missing_tool' + | 'missing_test_controller' + | 'unsatisfied_contract' + /** Legacy values kept for backward compatibility — map to the contract set above. */ + | 'not_testable' + | 'dependency_failed' + | 'missing_test_harness'; + export interface StoryboardStepPreview { step_id: string; phase_id: string; @@ -259,14 +334,15 @@ export interface StoryboardStepResult { passed: boolean; /** True when the step was not executed */ skipped?: boolean; - /** Why the step was skipped */ - skip_reason?: - | 'not_testable' - | 'dependency_failed' - | 'missing_test_harness' - | 'missing_tool' - /** Storyboard predates the agent's declared adcp.major_versions. */ - | 'not_applicable'; + /** Why the step was skipped. See `StoryboardSkipReason` for contract semantics. */ + skip_reason?: StoryboardSkipReason; + /** + * Human-readable detail for the skip — MUST cite declared supported_protocols + * / specialisms (not_applicable), the missing tool (missing_tool), the + * prerequisite step id (prerequisite_failed), or the contract key + * (unsatisfied_contract). + */ + skip_detail?: string; /** True when the step expected an error (inverted pass/fail) */ expect_error?: boolean; duration_ms: number; @@ -275,6 +351,12 @@ export interface StoryboardStepResult { /** Accumulated context after this step */ context: StoryboardContext; error?: string; + /** Exact request the runner sent (populated when the step made a call). */ + request_record?: RecordedRequest; + /** Exact response the agent returned (populated when the step made a call). */ + response_record?: RecordedResponse; + /** MCP/A2A extraction path the runner used to parse the response. */ + extraction?: ExtractionRecord; /** Preview of the next step (for LLM consumption) */ next?: StoryboardStepPreview; /** Agent URL that served this step (multi-instance mode). Absent in single-URL mode. */ diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index 227949f14..8fb41a759 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -8,10 +8,11 @@ * - status_code: check the TaskResult status */ -import { TOOL_RESPONSE_SCHEMAS } from '../../utils/response-schemas'; +import { TOOL_RESPONSE_SCHEMAS, getResponseSchemaLocator } from '../../utils/response-schemas'; +import { ADCP_VERSION } from '../../version'; import type { TaskResult } from '../types'; import type { HttpProbeResult, StoryboardValidation, ValidationResult } from './types'; -import { resolvePath } from './path'; +import { resolvePath, toJsonPointer } from './path'; import { PROBE_TASK_ALLOWLIST } from './test-kit'; /** @@ -122,6 +123,7 @@ function validateResponseSchema( taskName: string, taskResult: TaskResult ): ValidationResult { + const locator = getResponseSchemaLocator(taskName, ADCP_VERSION); const schema = TOOL_RESPONSE_SCHEMAS[taskName]; if (!schema) { return { @@ -129,6 +131,8 @@ function validateResponseSchema( passed: false, description: validation.description, error: `No schema registered for task "${taskName}"`, + expected: locator?.schema_id, + ...(locator && { schema_id: locator.schema_id, schema_url: locator.schema_url }), }; } @@ -141,20 +145,37 @@ function validateResponseSchema( check: 'response_schema', passed: true, description: validation.description, + ...(locator && { schema_id: locator.schema_id, schema_url: locator.schema_url }), }; } - // Format Zod errors - const issues = parseResult.error.issues - .slice(0, 5) - .map(i => `${i.path.join('.')}: ${i.message}`) - .join('; '); + // AJV-style structured actual per runner-output contract. + const actual = parseResult.error.issues.slice(0, 20).map(i => ({ + instance_path: i.path.length > 0 ? '/' + i.path.map(p => String(p).replace(/~/g, '~0').replace(/\//g, '~1')).join('/') : '', + keyword: i.code, + message: i.message, + })); + + // Pin the json_pointer to the first offending field for the actionability + // "one-line read" the contract calls out. Multiple issues are still in `actual`. + const firstIssue = parseResult.error.issues[0]; + const json_pointer = + firstIssue && firstIssue.path.length > 0 + ? '/' + firstIssue.path.map(p => String(p).replace(/~/g, '~0').replace(/\//g, '~1')).join('/') + : ''; return { check: 'response_schema', passed: false, description: validation.description, - error: issues, + error: parseResult.error.issues + .slice(0, 5) + .map(i => `${i.path.join('.')}: ${i.message}`) + .join('; '), + json_pointer, + expected: locator?.schema_id, + actual, + ...(locator && { schema_id: locator.schema_id, schema_url: locator.schema_url }), }; } @@ -181,6 +202,9 @@ function validateFieldPresent(validation: StoryboardValidation, taskResult: Task passed: present, description: validation.description, path: validation.path, + json_pointer: toJsonPointer(validation.path), + expected: 'present', + actual: present ? value : null, error: present ? undefined : `Field not found at path: ${validation.path}`, }; } @@ -208,6 +232,7 @@ function validateFieldValue(validation: StoryboardValidation, taskResult: TaskRe } const actual = resolvePath(taskResult.data, validation.path); + const json_pointer = toJsonPointer(validation.path); // allowed_values: pass if actual matches any value in the list if (validation.allowed_values?.length) { @@ -217,6 +242,9 @@ function validateFieldValue(validation: StoryboardValidation, taskResult: TaskRe passed, description: validation.description, path: validation.path, + json_pointer, + expected: validation.allowed_values, + actual, error: passed ? undefined : `Expected one of ${JSON.stringify(validation.allowed_values)}, got ${JSON.stringify(actual)}`, @@ -231,6 +259,9 @@ function validateFieldValue(validation: StoryboardValidation, taskResult: TaskRe passed, description: validation.description, path: validation.path, + json_pointer, + expected: validation.value, + actual, error: passed ? undefined : `Expected ${JSON.stringify(validation.value)}, got ${JSON.stringify(actual)}`, }; } @@ -247,6 +278,8 @@ function validateStatusCode(validation: StoryboardValidation, taskResult: TaskRe check: 'status_code', passed, description: validation.description, + expected: 'success', + actual: passed ? 'success' : 'failed', error: passed ? undefined : `Task failed: ${taskResult.error || 'unknown error'}`, }; } @@ -277,12 +310,15 @@ function validateErrorCode(validation: StoryboardValidation, taskResult: TaskRes extractCodeFromErrorString(taskResult.error); if (validation.allowed_values?.length) { - const actual = errorCode !== undefined && errorCode !== null ? String(errorCode) : undefined; - const passed = actual !== undefined && validation.allowed_values.some(v => String(v) === actual); + const actualStr = errorCode !== undefined && errorCode !== null ? String(errorCode) : undefined; + const passed = actualStr !== undefined && validation.allowed_values.some(v => String(v) === actualStr); return { check: 'error_code', passed, description: validation.description, + json_pointer: '/adcp_error/code', + expected: validation.allowed_values, + actual: errorCode ?? null, error: passed ? undefined : `Expected one of ${JSON.stringify(validation.allowed_values)}, got ${JSON.stringify(errorCode)}`, @@ -296,6 +332,9 @@ function validateErrorCode(validation: StoryboardValidation, taskResult: TaskRes check: 'error_code', passed: hasCode, description: validation.description, + json_pointer: '/adcp_error/code', + expected: 'present', + actual: errorCode ?? null, error: hasCode ? undefined : 'No error code found in response', }; } @@ -305,6 +344,9 @@ function validateErrorCode(validation: StoryboardValidation, taskResult: TaskRes check: 'error_code', passed, description: validation.description, + json_pointer: '/adcp_error/code', + expected: validation.value, + actual: errorCode ?? null, error: passed ? undefined : `Expected error code "${validation.value}", got "${errorCode}"`, }; } @@ -320,6 +362,8 @@ function validateHttpStatus(validation: StoryboardValidation, hr: HttpProbeResul check: 'http_status', passed, description: validation.description, + expected, + actual: hr.status, error: passed ? undefined : `Expected HTTP ${expected}, got ${hr.status}${hr.error ? ` (${hr.error})` : ''}`, }; } @@ -328,7 +372,13 @@ function validateHttpStatusIn(validation: StoryboardValidation, hr: HttpProbeRes const allowed = Array.isArray(validation.allowed_values) ? validation.allowed_values : []; const passed = allowed.some(v => v === hr.status); if (passed) { - return { check: 'http_status_in', passed: true, description: validation.description }; + return { + check: 'http_status_in', + passed: true, + description: validation.description, + expected: allowed, + actual: hr.status, + }; } // Disambiguate two failure modes that produce the same HTTP-status mismatch: // (a) the kit's `probe_task` requires non-empty params, so the agent 400s on @@ -344,6 +394,8 @@ function validateHttpStatusIn(validation: StoryboardValidation, hr: HttpProbeRes check: 'http_status_in', passed: false, description: validation.description, + expected: allowed, + actual: hr.status, error: `Agent returned HTTP ${hr.status} with a schema-validation body before any auth response. ` + `Two possible causes: (1) \`test_kit.auth.probe_task\` points at a task that requires ` + @@ -357,6 +409,8 @@ function validateHttpStatusIn(validation: StoryboardValidation, hr: HttpProbeRes check: 'http_status_in', passed: false, description: validation.description, + expected: allowed, + actual: hr.status, error: `Expected HTTP status in ${JSON.stringify(allowed)}, got ${hr.status}${hr.error ? ` (${hr.error})` : ''}`, }; } @@ -417,7 +471,13 @@ function validateOn401RequireHeader(validation: StoryboardValidation, hr: HttpPr // Silent pass when the response isn't a 401 — the conditional is part of // the spec (RFC 6750 §3 only applies to 401s). if (hr.status !== 401) { - return { check: 'on_401_require_header', passed: true, description: validation.description }; + return { + check: 'on_401_require_header', + passed: true, + description: validation.description, + expected: 'not_applicable', + actual: hr.status, + }; } const value = hr.headers[header]; const passed = typeof value === 'string' && value.length > 0; @@ -425,6 +485,9 @@ function validateOn401RequireHeader(validation: StoryboardValidation, hr: HttpPr check: 'on_401_require_header', passed, description: validation.description, + json_pointer: `/headers/${header}`, + expected: `non-empty ${header} header`, + actual: value ?? null, error: passed ? undefined : `401 response missing required header "${header}".`, }; } @@ -463,9 +526,9 @@ function validateResourceEqualsAgentUrl( error: 'Response body missing string `resource` field.', }; } - const expected = normalizeAgentUrl(agentUrl); - const actual = normalizeAgentUrl(resource); - const passed = actual === expected; + const expectedUrl = normalizeAgentUrl(agentUrl); + const actualUrl = normalizeAgentUrl(resource); + const passed = actualUrl === expectedUrl; // Don't echo the advertised value verbatim — compliance reports may be // shared publicly and the raw diff helps attackers probe victim agents. // Surface just enough for the operator to self-diagnose: their own URL, @@ -478,10 +541,10 @@ function validateResourceEqualsAgentUrl( } catch { /* ignore */ } - const expectedHost = new URL(expected).host; + const expectedHost = new URL(expectedUrl).host; const hostDiffers = actualHost !== expectedHost; redactedError = - `RFC 9728 \`resource\` does not equal the URL clients call (${expected}). ` + + `RFC 9728 \`resource\` does not equal the URL clients call (${expectedUrl}). ` + (hostDiffers ? `Advertised host differs from the agent host — the most common cause is copying your authorization server origin into \`resource\`. ` : `Advertised path differs from the agent path. `) + @@ -491,6 +554,11 @@ function validateResourceEqualsAgentUrl( check: 'resource_equals_agent_url', passed, description: validation.description, + json_pointer: '/resource', + expected: expectedUrl, + // Pass through the host-diff signal only; don't echo the raw advertised + // URL to avoid leaking victim-agent probe targets in shared reports. + actual: passed ? expectedUrl : '', error: redactedError, }; } @@ -506,6 +574,8 @@ function validateAnyOf(validation: StoryboardValidation, contributions: Set> = { z.object({ errors: z.array(schemas.ErrorSchema) }).passthrough(), ]), }; + +/** + * Map of AdCP task names to the schema-index subdirectory that holds their + * response schema. Drives the runner-output contract's `schema_id` / `schema_url` + * fields so compliance reports tell implementors exactly which artifact to + * re-validate against. + * + * The `$id` emitted for task X is `/schemas/{version}/{subdir}/{kebab(X)}-response.json`, + * matching the convention in schemas/cache/{version}/index.json. + */ +export const TOOL_SCHEMA_SUBDIR: Partial> = { + // Media buy + get_products: 'media-buy', + create_media_buy: 'media-buy', + update_media_buy: 'media-buy', + get_media_buys: 'media-buy', + get_media_buy_delivery: 'media-buy', + provide_performance_feedback: 'media-buy', + list_creative_formats: 'media-buy', + sync_event_sources: 'media-buy', + log_event: 'media-buy', + sync_audiences: 'media-buy', + sync_catalogs: 'media-buy', + get_media_buy_artifacts: 'media-buy', + + // Creative + build_creative: 'creative', + preview_creative: 'creative', + sync_creatives: 'creative', + list_creatives: 'creative', + get_creative_delivery: 'creative', + get_creative_features: 'creative', + + // Signals + get_signals: 'signals', + activate_signal: 'signals', + + // Account + sync_accounts: 'account', + list_accounts: 'account', + sync_governance: 'account', + report_usage: 'account', + get_account_financials: 'account', + + // Governance — property lists, content standards, campaign governance + create_property_list: 'governance', + get_property_list: 'governance', + update_property_list: 'governance', + list_property_lists: 'governance', + delete_property_list: 'governance', + list_content_standards: 'content-standards', + get_content_standards: 'content-standards', + create_content_standards: 'content-standards', + update_content_standards: 'content-standards', + calibrate_content: 'content-standards', + validate_content_delivery: 'content-standards', + validate_property_delivery: 'governance', + sync_plans: 'governance', + check_governance: 'governance', + report_plan_outcome: 'governance', + get_plan_audit_logs: 'governance', + + // Collection lists + create_collection_list: 'collection', + update_collection_list: 'collection', + get_collection_list: 'collection', + list_collection_lists: 'collection', + delete_collection_list: 'collection', + + // Sponsored Intelligence + si_get_offering: 'si', + si_initiate_session: 'si', + si_send_message: 'si', + si_terminate_session: 'si', + + // Capabilities + get_adcp_capabilities: 'protocol', + + // Test controller + comply_test_controller: 'compliance', + + // Brand rights + get_brand_identity: 'brand', + get_rights: 'brand', + acquire_rights: 'brand', + update_rights: 'brand', + creative_approval: 'brand', +}; + +/** + * Task-name kebabization matching the AdCP schema index (`{task}-response.json`). + */ +function toKebab(taskName: string): string { + return taskName.replace(/_/g, '-'); +} + +/** + * Return `{ schema_id, schema_url }` for a task's response schema, or `undefined` + * when the task has no registered subdirectory. + * + * `schema_id` is the JSON-Schema `$id` convention used in schemas/cache/{version}/index.json. + * `schema_url` is a resolvable URL an implementor can fetch to re-validate locally. + */ +export function getResponseSchemaLocator( + taskName: string, + adcpVersion: string +): { schema_id: string; schema_url: string } | undefined { + const subdir = TOOL_SCHEMA_SUBDIR[taskName]; + if (!subdir) return undefined; + const file = `${toKebab(taskName)}-response.json`; + return { + schema_id: `/schemas/${adcpVersion}/${subdir}/${file}`, + schema_url: `https://adcontextprotocol.org/schemas/${adcpVersion}/${subdir}/${file}`, + }; +} diff --git a/test/lib/runner-output-contract.test.js b/test/lib/runner-output-contract.test.js new file mode 100644 index 000000000..256d8dc60 --- /dev/null +++ b/test/lib/runner-output-contract.test.js @@ -0,0 +1,191 @@ +// Runner-output contract conformance tests for #599. +// +// Every failed validation result MUST carry json_pointer / expected / actual, +// plus schema_id / schema_url for response_schema checks. The step result +// MUST carry an extraction.path and a recorded request/response. + +const { describe, it } = require('node:test'); +const assert = require('node:assert'); +const { runValidations } = require('../../dist/lib/testing/storyboard/validations'); +const { toJsonPointer } = require('../../dist/lib/testing/storyboard/path'); +const { getResponseSchemaLocator } = require('../../dist/lib/utils/response-schemas'); +const { getScenarioSkips } = require('../../dist/lib/testing/orchestrator'); + +function runOne(validations, taskResult, extras = {}) { + return runValidations(validations, { + taskName: extras.taskName ?? 'get_products', + taskResult, + agentUrl: extras.agentUrl ?? 'https://example.com/mcp', + contributions: extras.contributions ?? new Set(), + ...(extras.httpResult && { httpResult: extras.httpResult }), + }); +} + +describe('toJsonPointer', () => { + it('converts dot paths to RFC 6901 pointers', () => { + assert.strictEqual(toJsonPointer('adcp.idempotency'), '/adcp/idempotency'); + assert.strictEqual(toJsonPointer('accounts[0].account_id'), '/accounts/0/account_id'); + assert.strictEqual(toJsonPointer('status'), '/status'); + }); + + it('escapes ~ and / per RFC 6901', () => { + assert.strictEqual(toJsonPointer('a~b'), '/a~0b'); + // parsePath treats "/" as a segment separator, so a literal "/" can't appear + // inside a single segment — but escaping still fires when we pass it in via + // a manual pseudo-segment. + }); +}); + +describe('getResponseSchemaLocator', () => { + it('produces an $id under the correct subdirectory', () => { + const loc = getResponseSchemaLocator('get_adcp_capabilities', 'latest'); + assert.strictEqual(loc.schema_id, '/schemas/latest/protocol/get-adcp-capabilities-response.json'); + assert.ok(loc.schema_url.endsWith('/schemas/latest/protocol/get-adcp-capabilities-response.json')); + assert.ok(loc.schema_url.startsWith('https://adcontextprotocol.org/')); + }); + + it('maps get_products to media-buy', () => { + const loc = getResponseSchemaLocator('get_products', 'latest'); + assert.strictEqual(loc.schema_id, '/schemas/latest/media-buy/get-products-response.json'); + }); + + it('returns undefined for unknown tasks', () => { + assert.strictEqual(getResponseSchemaLocator('no_such_task', 'latest'), undefined); + }); +}); + +describe('validateResponseSchema emits contract fields', () => { + it('attaches schema_id and schema_url on pass', () => { + const taskResult = { + success: true, + data: { + adcp: { + major_versions: [3], + idempotency: { replay_ttl_seconds: 86400 }, + }, + supported_protocols: ['media_buy'], + }, + }; + const [r] = runOne( + [{ check: 'response_schema', description: 'matches capabilities schema' }], + taskResult, + { taskName: 'get_adcp_capabilities' } + ); + assert.strictEqual(r.passed, true, r.error); + assert.ok(r.schema_id, 'schema_id missing'); + assert.ok(r.schema_url, 'schema_url missing'); + }); + + it('emits json_pointer + AJV-style actual on failure', () => { + const taskResult = { + success: true, + // Missing required `adcp.idempotency` — should fail the schema. + data: { adcp: { major_versions: [3] } }, + }; + const [r] = runOne( + [{ check: 'response_schema', description: 'matches capabilities schema' }], + taskResult, + { taskName: 'get_adcp_capabilities' } + ); + assert.strictEqual(r.passed, false); + assert.ok(r.json_pointer?.startsWith('/'), `expected RFC 6901 pointer, got ${r.json_pointer}`); + assert.ok(Array.isArray(r.actual), 'actual should be AJV-style array'); + assert.ok(r.actual.length > 0); + assert.ok('instance_path' in r.actual[0]); + assert.ok('message' in r.actual[0]); + assert.strictEqual(r.expected, r.schema_id, 'expected should be the schema $id'); + }); +}); + +describe('validateFieldPresent emits contract fields', () => { + it('emits json_pointer, expected=present, actual=null on miss', () => { + const [r] = runOne( + [{ check: 'field_present', path: 'adcp.idempotency', description: 'idempotency is declared' }], + { success: true, data: { adcp: {} } } + ); + assert.strictEqual(r.passed, false); + assert.strictEqual(r.json_pointer, '/adcp/idempotency'); + assert.strictEqual(r.expected, 'present'); + assert.strictEqual(r.actual, null); + }); + + it('emits the observed value on pass', () => { + const [r] = runOne( + [{ check: 'field_present', path: 'adcp.idempotency.replay_ttl_seconds', description: 'ttl present' }], + { success: true, data: { adcp: { idempotency: { replay_ttl_seconds: 86400 } } } } + ); + assert.strictEqual(r.passed, true); + assert.strictEqual(r.actual, 86400); + }); +}); + +describe('validateFieldValue emits contract fields', () => { + it('emits expected/actual on exact-match failure', () => { + const [r] = runOne( + [{ check: 'field_value', path: 'status', value: 'completed', description: 'status is completed' }], + { success: true, data: { status: 'working' } } + ); + assert.strictEqual(r.passed, false); + assert.strictEqual(r.json_pointer, '/status'); + assert.strictEqual(r.expected, 'completed'); + assert.strictEqual(r.actual, 'working'); + }); + + it('emits allowed_values as expected on list-match failure', () => { + const [r] = runOne( + [ + { + check: 'field_value', + path: 'status', + allowed_values: ['completed', 'submitted'], + description: 'status in terminal set', + }, + ], + { success: true, data: { status: 'working' } } + ); + assert.strictEqual(r.passed, false); + assert.deepStrictEqual(r.expected, ['completed', 'submitted']); + assert.strictEqual(r.actual, 'working'); + }); +}); + +describe('validateErrorCode emits contract fields', () => { + it('emits json_pointer=/adcp_error/code and the observed code', () => { + const [r] = runOne( + [{ check: 'error_code', value: 'INVALID_REQUEST', description: 'expected INVALID_REQUEST' }], + { success: false, data: { adcp_error: { code: 'VALIDATION_ERROR' } }, error: 'VALIDATION_ERROR: bad' } + ); + assert.strictEqual(r.passed, false); + assert.strictEqual(r.json_pointer, '/adcp_error/code'); + assert.strictEqual(r.expected, 'INVALID_REQUEST'); + assert.strictEqual(r.actual, 'VALIDATION_ERROR'); + }); +}); + +describe('getScenarioSkips', () => { + it('distinguishes missing_tool from missing_test_controller', () => { + // Explicit filter — deterministic_* are excluded from DEFAULT_SCENARIOS. + const skips = getScenarioSkips(['get_products'], [ + 'deterministic_media_buy', + 'create_media_buy', + 'health_check', + ]); + const detMediaBuy = skips.find(s => s.scenario === 'deterministic_media_buy'); + assert.ok(detMediaBuy, 'deterministic_media_buy should be skipped'); + assert.strictEqual(detMediaBuy.reason, 'missing_test_controller'); + assert.ok(detMediaBuy.detail.includes('comply_test_controller'), detMediaBuy.detail); + + const createMediaBuy = skips.find(s => s.scenario === 'create_media_buy'); + assert.ok(createMediaBuy); + assert.strictEqual(createMediaBuy.reason, 'missing_tool'); + assert.ok(createMediaBuy.detail.includes('create_media_buy'), createMediaBuy.detail); + + const healthCheck = skips.find(s => s.scenario === 'health_check'); + assert.strictEqual(healthCheck, undefined, 'health_check is always applicable'); + }); + + it('returns empty when every scenario is applicable', () => { + const skips = getScenarioSkips([], ['health_check']); + assert.strictEqual(skips.length, 0); + }); +});