diff --git a/.changeset/format-mismatch-hint.md b/.changeset/format-mismatch-hint.md new file mode 100644 index 000000000..89617f5b7 --- /dev/null +++ b/.changeset/format-mismatch-hint.md @@ -0,0 +1,7 @@ +--- +"@adcp/client": minor +--- + +feat(conformance): add FormatMismatchHint to StoryboardStepHint taxonomy + +Third member of the `StoryboardStepHint` discriminated union. Emits a structured hint when a response_schema validation passes the lenient Zod check but fails strict AJV validation on a `format` keyword (`date-time`, `uuid`, `uri`, `email`, etc.). Non-fatal — does not flip step pass/fail. Fires only on strict_only_failure steps so it surfaces the strict/lenient delta without adding noise to already-failing validations. Closes #947. diff --git a/src/lib/testing/index.ts b/src/lib/testing/index.ts index e0cb2e6e2..077460a8a 100644 --- a/src/lib/testing/index.ts +++ b/src/lib/testing/index.ts @@ -264,6 +264,7 @@ export { type StoryboardStepResult, type StoryboardStepHint, type ContextValueRejectedHint, + type FormatMismatchHint, type ContextProvenanceEntry, type StoryboardPhaseResult, type StoryboardResult, diff --git a/src/lib/testing/storyboard/format-mismatch-hints.ts b/src/lib/testing/storyboard/format-mismatch-hints.ts new file mode 100644 index 000000000..5c53a0541 --- /dev/null +++ b/src/lib/testing/storyboard/format-mismatch-hints.ts @@ -0,0 +1,119 @@ +/** + * Format-mismatch hint detection. + * + * When a response_schema validation passes the lenient Zod check but fails + * strict AJV validation on a `format` keyword (`date-time`, `uuid`, `uri`, + * `email`, etc.), emit a non-fatal `format_mismatch` hint. This is the + * strict/lenient delta the runner already computes inside + * `StrictValidationVerdict` — today the delta shows up as a `warning` string + * on `ValidationResult`; this module surfaces it as machine-readable fields + * a renderer can quote back at the developer with an exact fix suggestion. + * + * Fires only on `strict_only_failure` steps: lenient Zod accepted the + * response (`ValidationResult.passed === true`) but strict AJV rejected it + * (`strict.valid === false`). Firing on already-failing lenient steps would + * add noise on top of more fundamental failures. See issue #947. + */ + +import type { FormatMismatchHint, ValidationResult } from './types'; +import { resolvePath } from './path'; + +const FORMAT_KEYWORD_RE = /must match format "([^"]+)"/i; +const OBSERVED_VALUE_MAX_LEN = 200; + +/** + * Extract `format_mismatch` hints from a step's validation results. + * + * Pure — safe to call regardless of step pass/fail. Returns an empty array + * when no strict_only_failure response_schema validations have format issues. + * + * @param validations - The step's computed validation results. + * @param tool - AdCP task name (snake_case) the step dispatched. + * @param data - Raw parsed response payload (`taskResult.data`). Used to + * extract `observed_value` at the RFC 6901 pointer AJV reported. + */ +export function detectFormatMismatchHints( + validations: ValidationResult[], + tool: string, + data?: unknown +): FormatMismatchHint[] { + const hints: FormatMismatchHint[] = []; + for (const v of validations) { + if (v.check !== 'response_schema') continue; + if (!v.passed) continue; // strict_only_failure only — lenient must have passed + if (!v.strict || v.strict.valid) continue; // no AJV failure to report + for (const issue of v.strict.issues ?? []) { + if (issue.keyword !== 'format') continue; + const instance_path = issue.instance_path; + // Fallback to '(unknown)' rather than issue.keyword ('format') — the keyword + // filter already guarantees keyword === 'format', so using it as a fallback + // produces the confusing message `must match format "format"`. + const expected_format = extractFormatName(issue.message) ?? '(unknown)'; + const observed_value = extractObservedValue(data, instance_path); + hints.push({ + kind: 'format_mismatch', + message: buildMessage(tool, instance_path, expected_format, observed_value), + tool, + instance_path, + expected_format, + ...(observed_value !== undefined && { observed_value }), + }); + } + } + return hints; +} + +/** + * Parse the format name from an AJV format-keyword message. + * AJV's stable template: `must match format ""`. + * Falls back to undefined when the message doesn't match (AJV version skew + * or a custom format validator with non-standard message). + */ +function extractFormatName(message: string): string | undefined { + return FORMAT_KEYWORD_RE.exec(message)?.[1]; +} + +/** + * Convert an RFC 6901 JSON Pointer to the dot-bracket notation `resolvePath` + * speaks. `/packages/0/start_date` → `packages[0].start_date`. + * Applies RFC 6901 escape decoding (`~1` → `/`, `~0` → `~`). + */ +function pointerToDotPath(pointer: string): string { + if (!pointer.startsWith('/')) return pointer; + return pointer + .slice(1) + .split('/') + .map(seg => seg.replace(/~1/g, '/').replace(/~0/g, '~')) // RFC 6901 §3: decode ~1 before ~0 + .map(seg => (/^\d+$/.test(seg) ? `[${seg}]` : seg)) + .join('.') + .replace(/\.\[/g, '['); +} + +/** + * Resolve the value at `pointer` inside `data` and return it if it is a + * string within the length cap. Non-string values (objects, arrays, numbers) + * are not echoed — binary blobs and structured payloads would inflate the hint + * and risk leaking sensitive nested data. Truncates long strings at 200 chars + * using code-point slicing so surrogate pairs aren't cleaved. + */ +function extractObservedValue(data: unknown, pointer: string): string | undefined { + if (data === undefined || data === null) return undefined; + const dotPath = pointerToDotPath(pointer); + const value = resolvePath(data, dotPath); + if (typeof value !== 'string') return undefined; + if (value.length <= OBSERVED_VALUE_MAX_LEN) return value; + return ( + Array.from(value) + .slice(0, OBSERVED_VALUE_MAX_LEN - 1) + .join('') + '…' + ); +} + +function buildMessage(tool: string, instance_path: string, expected_format: string, observed_value?: string): string { + const fieldRepr = instance_path || '/'; + const valueClause = observed_value !== undefined ? ` (observed: "${observed_value}")` : ''; + return ( + `\`${tool}\` response: \`${fieldRepr}\` must match format "${expected_format}"${valueClause}. ` + + `Strict JSON-schema (AJV) rejected this value; lenient Zod validation accepted it.` + ); +} diff --git a/src/lib/testing/storyboard/index.ts b/src/lib/testing/storyboard/index.ts index 5b5aeb238..9d98b5093 100644 --- a/src/lib/testing/storyboard/index.ts +++ b/src/lib/testing/storyboard/index.ts @@ -32,6 +32,7 @@ export type { ContextInput, ContextProvenanceEntry, ContextValueRejectedHint, + FormatMismatchHint, StoryboardContext, StoryboardRunOptions, ValidationResult, @@ -137,6 +138,9 @@ export type { ContextWriteResult } from './context'; // Rejection-hint detection (issue #870) export { detectContextRejectionHints } from './rejection-hints'; +// Format-mismatch hint detection (issue #947) +export { detectFormatMismatchHints } from './format-mismatch-hints'; + // Request builder export { buildRequest, hasRequestBuilder } from './request-builder'; diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 50245c793..7f6aec38e 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -19,6 +19,7 @@ import { type RunnerVariables, } from './context'; import { detectContextRejectionHints } from './rejection-hints'; +import { detectFormatMismatchHints } from './format-mismatch-hints'; import { runValidations, type ValidationContext } from './validations'; import { enrichRequest, hasRequestEnricher } from './request-builder'; import { resolveAccount, resolveBrand } from '../client'; @@ -1685,10 +1686,23 @@ async function executeStep( // since the rejected value can't have come from this step's own // extraction. const stepFailed = !(passed && allValidationsPassed); - const hints = + // context_value_rejected hints: fire only on step failure (the seller's + // error response carries the accepted-values list we trace back to a + // prior-step context write). Gate unchanged from pre-#947 behavior. + const contextRejectionHints = stepFailed && runState.contextProvenance ? detectContextRejectionHints(taskResult, request, context, runState.contextProvenance, effectiveStep.task) : []; + // format_mismatch hints: fire on strict_only_failure validations (lenient + // Zod passed, strict AJV rejected on a format keyword). These are most + // valuable when the step is green — AJV found a silent strictness gap the + // step would otherwise surface only as a warning string. Gate is separate + // from stepFailed so hints reach lenient-passing steps. Issue #947. + // No explicit guard needed: skipped / probe / webhook-assertion steps have + // empty `validations` arrays, so detectFormatMismatchHints returns [] for them + // without inspecting taskResult. + const formatMismatchHints = detectFormatMismatchHints(validations, effectiveStep.task, taskResult?.data); + const hints = [...contextRejectionHints, ...formatMismatchHints]; // Build next step preview const next = getNextStepPreview(step.id, allSteps, updatedContext, runState.runnerVars); diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index c152ef18d..f3e06c377 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -1049,12 +1049,46 @@ export interface ContextProvenanceEntry { } /** - * Non-fatal hint attached to a step result. Today the runner only emits - * `context_value_rejected` — more kinds may be added over time. The - * discriminator lives on `kind` so consumers that only know how to render - * a subset can ignore the rest without losing them. + * Non-fatal hint attached to a step result. More kinds may be added over + * time. The discriminator lives on `kind` so consumers that only know how + * to render a subset can ignore the rest without losing them. */ -export type StoryboardStepHint = ContextValueRejectedHint; +export type StoryboardStepHint = ContextValueRejectedHint | FormatMismatchHint; + +/** + * A response field passed the lenient Zod check but failed strict JSON-schema + * (AJV) validation on a `format` keyword (`date-time`, `uuid`, `uri`, `email`, + * etc.). Fires only on strict_only_failure validations — lenient passed, strict + * failed — so it adds signal on top of a green step without obscuring hard + * failures that already surface via `ValidationResult.error`. + * + * Non-fatal: does not flip the step's pass/fail. `message` carries a + * human-readable summary renderers can display without needing to understand + * the structured fields. + */ +export interface FormatMismatchHint { + kind: 'format_mismatch'; + /** Pre-formatted human-readable summary suitable for a console line. */ + message: string; + /** AdCP tool name (snake_case) whose response contained the violation. */ + tool: string; + /** + * RFC 6901 JSON Pointer to the offending field in the agent's response. + * Matches `SchemaValidationError.instance_path` — `FormatMismatchHint` is + * a typed projection of a `SchemaValidationError` filtered by + * `keyword === 'format'`. + */ + instance_path: string; + /** JSON Schema `format` keyword value that rejected the field (e.g. `'date-time'`, `'uuid'`, `'uri'`). */ + expected_format: string; + /** + * The value the runner observed at `instance_path`. Present only when the + * value is a string — non-string values (objects, arrays, numbers) are not + * echoed into hints. Truncated at 200 characters so binary blobs and + * unexpectedly long strings don't inflate the hint payload. + */ + observed_value?: string; +} /** * A seller rejected a request value that the runner traced back to a diff --git a/src/lib/version.ts b/src/lib/version.ts index d1943ab1a..79d836206 100644 --- a/src/lib/version.ts +++ b/src/lib/version.ts @@ -4,7 +4,7 @@ /** * AdCP client library version */ -export const LIBRARY_VERSION = '5.16.0'; +export const LIBRARY_VERSION = '5.17.0'; /** * AdCP specification version this library is built for @@ -27,10 +27,10 @@ export const COMPATIBLE_ADCP_VERSIONS = ['v2.5', 'v2.6', 'v3', '3.0.0-beta.1', ' * Full version information */ export const VERSION_INFO = { - library: '5.16.0', + library: '5.17.0', adcp: '3.0.0', compatibleVersions: COMPATIBLE_ADCP_VERSIONS, - generatedAt: '2026-04-24T21:53:15.899Z', + generatedAt: '2026-04-25T10:06:32.349Z', } as const; /** diff --git a/test/lib/storyboard-format-mismatch-hints.test.js b/test/lib/storyboard-format-mismatch-hints.test.js new file mode 100644 index 000000000..bc8e8204c --- /dev/null +++ b/test/lib/storyboard-format-mismatch-hints.test.js @@ -0,0 +1,225 @@ +/** + * Tests for detectFormatMismatchHints — the format-mismatch hint detector + * that fires when lenient Zod passes but strict AJV rejects on a format + * keyword (issue #947). + * + * Covers: basic detection, strict_only_failure gate, observed_value extraction + * (including RFC 6901 pointer decoding and length truncation), expected_format + * parsing, and the message fallback when the AJV message template doesn't match. + */ + +const { test, describe } = require('node:test'); +const assert = require('node:assert/strict'); +const { detectFormatMismatchHints } = require('../../dist/lib/testing/storyboard/format-mismatch-hints'); + +function makeStrictOnlyFailure(instancePath, message, tool = 'create_media_buy') { + return [ + { + check: 'response_schema', + passed: true, // lenient Zod accepted + description: 'response matches schema', + strict: { + valid: false, + variant: 'sync', + issues: [ + { + instance_path: instancePath, + schema_path: '#/properties/start_date/format', + keyword: 'format', + message, + }, + ], + }, + }, + ]; +} + +describe('detectFormatMismatchHints', () => { + test('emits hint on strict_only_failure with format keyword', () => { + const validations = makeStrictOnlyFailure('/start_date', 'must match format "date-time"'); + const hints = detectFormatMismatchHints(validations, 'create_media_buy', { start_date: '2026-04-25' }); + + assert.equal(hints.length, 1); + const h = hints[0]; + assert.equal(h.kind, 'format_mismatch'); + assert.equal(h.tool, 'create_media_buy'); + assert.equal(h.instance_path, '/start_date'); + assert.equal(h.expected_format, 'date-time'); + assert.equal(h.observed_value, '2026-04-25'); + assert.ok(h.message.includes('date-time')); + assert.ok(h.message.includes('start_date')); + assert.ok(h.message.includes('2026-04-25')); + }); + + test('does not emit hint when lenient Zod also failed (strict_only_failure gate)', () => { + const validations = [ + { + check: 'response_schema', + passed: false, // lenient also failed — no hint + description: 'response matches schema', + strict: { + valid: false, + variant: 'sync', + issues: [ + { + instance_path: '/expires_at', + schema_path: '#/properties/expires_at/format', + keyword: 'format', + message: 'must match format "date-time"', + }, + ], + }, + }, + ]; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', { expires_at: 'not-a-date' }); + assert.equal(hints.length, 0); + }); + + test('does not emit hint when strict AJV passed', () => { + const validations = [ + { + check: 'response_schema', + passed: true, + description: 'response matches schema', + strict: { valid: true, variant: 'sync' }, + }, + ]; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', { start_date: '2026-04-25T00:00:00Z' }); + assert.equal(hints.length, 0); + }); + + test('does not emit hint for non-format AJV issues', () => { + const validations = [ + { + check: 'response_schema', + passed: true, + strict: { + valid: false, + variant: 'sync', + issues: [ + { + instance_path: '/name', + schema_path: '#/properties/name/maxLength', + keyword: 'maxLength', // not 'format' + message: 'must NOT have more than 255 characters', + }, + ], + }, + description: 'response matches schema', + }, + ]; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', {}); + assert.equal(hints.length, 0); + }); + + test('emits one hint per format issue when multiple issues present', () => { + const validations = [ + { + check: 'response_schema', + passed: true, + description: 'response matches schema', + strict: { + valid: false, + variant: 'sync', + issues: [ + { + instance_path: '/start_date', + schema_path: '#/properties/start_date/format', + keyword: 'format', + message: 'must match format "date-time"', + }, + { + instance_path: '/advertiser_id', + schema_path: '#/properties/advertiser_id/format', + keyword: 'format', + message: 'must match format "uuid"', + }, + ], + }, + }, + ]; + const data = { start_date: '2026-04-25', advertiser_id: 'not-a-uuid' }; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', data); + assert.equal(hints.length, 2); + const kinds = new Set(hints.map(h => h.expected_format)); + assert.ok(kinds.has('date-time')); + assert.ok(kinds.has('uuid')); + }); + + test('RFC 6901 pointer decoded correctly: ~1 and ~0 escapes', () => { + // /prop~0with~1slash → prop~with/slash key (RFC 6901 §3: ~1 before ~0) + const validations = makeStrictOnlyFailure('/prop~0with~1slash', 'must match format "uri"'); + const data = { 'prop~with/slash': 'not-a-uri' }; + const hints = detectFormatMismatchHints(validations, 'sync_creatives', data); + assert.equal(hints.length, 1); + assert.equal(hints[0].instance_path, '/prop~0with~1slash'); + assert.equal(hints[0].observed_value, 'not-a-uri'); + }); + + test('RFC 6901 pointer with array index', () => { + const validations = makeStrictOnlyFailure('/packages/0/start_date', 'must match format "date-time"'); + const data = { packages: [{ start_date: '2026-04-25' }] }; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', data); + assert.equal(hints.length, 1); + assert.equal(hints[0].observed_value, '2026-04-25'); + }); + + test('observed_value omitted for non-string values', () => { + const validations = makeStrictOnlyFailure('/count', 'must match format "integer"'); + const data = { count: 42 }; // number, not string + const hints = detectFormatMismatchHints(validations, 'create_media_buy', data); + assert.equal(hints.length, 1); + assert.equal(hints[0].observed_value, undefined); + }); + + test('observed_value truncated at 200 codepoints for long strings', () => { + // Build a string of exactly 201 'a' characters + const longValue = 'a'.repeat(201); + const validations = makeStrictOnlyFailure('/description', 'must match format "uri"'); + const data = { description: longValue }; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', data); + assert.equal(hints.length, 1); + const ov = hints[0].observed_value; + assert.ok(ov !== undefined); + // Should be 199 'a' + ellipsis (200 codepoints total) + assert.equal(Array.from(ov).length, 200); + assert.ok(ov.endsWith('…')); + }); + + test('expected_format falls back to (unknown) when AJV message does not match template', () => { + const validations = makeStrictOnlyFailure('/field', 'custom validator rejected value'); + const hints = detectFormatMismatchHints(validations, 'create_media_buy', {}); + assert.equal(hints.length, 1); + assert.equal(hints[0].expected_format, '(unknown)'); + assert.ok(hints[0].message.includes('(unknown)')); + }); + + test('returns empty array when validations is empty', () => { + const hints = detectFormatMismatchHints([], 'create_media_buy', {}); + assert.equal(hints.length, 0); + }); + + test('returns empty array when data is undefined', () => { + const validations = makeStrictOnlyFailure('/start_date', 'must match format "date-time"'); + const hints = detectFormatMismatchHints(validations, 'create_media_buy', undefined); + assert.equal(hints.length, 1); + assert.equal(hints[0].observed_value, undefined); + }); + + test('non-response_schema checks are ignored', () => { + const validations = [ + { + check: 'field_present', // not response_schema + passed: true, + description: 'field present', + strict: { + valid: false, + variant: 'sync', + issues: [{ instance_path: '/x', schema_path: '', keyword: 'format', message: 'must match format "uri"' }], + }, + }, + ]; + const hints = detectFormatMismatchHints(validations, 'create_media_buy', { x: 'not-a-uri' }); + assert.equal(hints.length, 0); + }); +});