diff --git a/.changeset/field-value-or-absent.md b/.changeset/field-value-or-absent.md new file mode 100644 index 000000000..816f80648 --- /dev/null +++ b/.changeset/field-value-or-absent.md @@ -0,0 +1,5 @@ +--- +"@adcp/client": minor +--- + +Added `field_value_or_absent` storyboard check matcher. Passes when the field is absent OR present with a value in `allowed_values` / matching `value`; fails only when present with a disallowed value. Use it for envelope-tolerant assertions (e.g. fresh-path `replayed`) where the spec allows omission but forbids a wrong value. Closes #873. diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index fb3c41a05..b4161ca89 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -371,6 +371,7 @@ export type StoryboardValidationCheck = | 'response_schema' | 'field_present' | 'field_value' + | 'field_value_or_absent' | 'status_code' | 'error_code' // HTTP-probe checks (for raw_probe tasks) diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index 6c7ebc13d..b34e8cb81 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -87,6 +87,8 @@ function runValidation(validation: StoryboardValidation, ctx: ValidationContext) return validateFieldPresent(validation, resolveTarget(ctx)); case 'field_value': return validateFieldValue(validation, resolveTarget(ctx)); + case 'field_value_or_absent': + return validateFieldValueOrAbsent(validation, resolveTarget(ctx)); case 'status_code': return requireTaskResult(ctx, validation, tr => validateStatusCode(validation, tr)); case 'error_code': @@ -724,6 +726,89 @@ function validateFieldValue(validation: StoryboardValidation, taskResult: TaskRe }; } +// ──────────────────────────────────────────────────────────── +// field_value_or_absent: envelope-tolerant variant of field_value +// +// Pass when the field is absent OR present-and-matching. Fail only when the +// field is present with a disallowed value. Lets a storyboard keep positive +// coverage on a spec-optional field without penalizing agents that omit it. +// Spec: adcontextprotocol/adcp #3013 envelope `replayed` semantics. +// ──────────────────────────────────────────────────────────── + +function validateFieldValueOrAbsent(validation: StoryboardValidation, taskResult: TaskResult): ValidationResult { + if (!validation.path) { + return { + check: 'field_value_or_absent', + passed: false, + description: validation.description, + path: validation.path, + error: 'No path specified for field_value_or_absent validation', + json_pointer: null, + expected: 'path must be set in storyboard validation entry', + actual: null, + }; + } + + const actual = resolvePath(taskResult.data, validation.path); + const pointer = toJsonPointer(validation.path); + + // Absent → pass. The check only fires when the field is present. + if (actual === undefined) { + return { + check: 'field_value_or_absent', + passed: true, + description: validation.description, + path: validation.path, + json_pointer: pointer, + }; + } + + // Present → fall through to the same value / allowed_values semantics as field_value. + if (validation.allowed_values?.length) { + const passed = validation.allowed_values.some(v => valuesMatch(actual, v)); + if (passed) { + return { + check: 'field_value_or_absent', + passed: true, + description: validation.description, + path: validation.path, + json_pointer: pointer, + }; + } + return { + check: 'field_value_or_absent', + passed: false, + description: validation.description, + path: validation.path, + error: `Expected absent or one of ${JSON.stringify(validation.allowed_values)}, got ${JSON.stringify(actual)}`, + json_pointer: pointer, + expected: validation.allowed_values, + actual, + }; + } + + const passed = valuesMatch(actual, validation.value); + if (passed) { + return { + check: 'field_value_or_absent', + passed: true, + description: validation.description, + path: validation.path, + json_pointer: pointer, + }; + } + return { + check: 'field_value_or_absent', + passed: false, + description: validation.description, + path: validation.path, + error: `Expected absent or ${JSON.stringify(validation.value)}, got ${JSON.stringify(actual)}`, + json_pointer: pointer, + expected: validation.value, + actual, + }; +} + // ──────────────────────────────────────────────────────────── // status_code: check TaskResult status // ──────────────────────────────────────────────────────────── diff --git a/test/lib/storyboard-drift.test.js b/test/lib/storyboard-drift.test.js index 57c932e6c..aab3cde80 100644 --- a/test/lib/storyboard-drift.test.js +++ b/test/lib/storyboard-drift.test.js @@ -1,9 +1,10 @@ /** * Schema drift detection for storyboard YAML validations. * - * Catches when field_present / field_value paths in storyboard YAML - * reference fields that don't exist in the corresponding Zod response - * schemas, and when context extractors reference tasks without schemas. + * Catches when field_present / field_value / field_value_or_absent paths in + * storyboard YAML reference fields that don't exist in the corresponding + * Zod response schemas, and when context extractors reference tasks without + * schemas. */ const { describe, it } = require('node:test'); @@ -136,7 +137,10 @@ function collectFieldValidations(storyboards) { const isErrorStep = step.validations.some(v => v.check === 'is_error'); if (isErrorStep) continue; for (const v of step.validations) { - if ((v.check === 'field_present' || v.check === 'field_value') && v.path) { + if ( + (v.check === 'field_present' || v.check === 'field_value' || v.check === 'field_value_or_absent') && + v.path + ) { if (ENVELOPE_PATHS.has(v.path)) continue; // protocol-level, not per-schema entries.push({ storyboard: sb.id, @@ -166,7 +170,10 @@ describe('storyboard schema drift', () => { }); it('found field validations to check', () => { - assert.ok(fieldValidations.length > 0, 'Expected at least one field_present or field_value validation'); + assert.ok( + fieldValidations.length > 0, + 'Expected at least one field_present, field_value, or field_value_or_absent validation' + ); }); // Drift entries cleared by upstream fixes that haven't shipped in the @@ -247,6 +254,27 @@ describe('storyboard schema drift', () => { } }); + describe('field_value_or_absent paths are reachable in response schemas', () => { + const tolerantValidations = fieldValidations.filter(v => v.check === 'field_value_or_absent'); + + for (const entry of tolerantValidations) { + const schema = TOOL_RESPONSE_SCHEMAS[entry.task]; + if (!schema) continue; + + const key = `${entry.storyboard}/${entry.step}:${entry.path}`; + const skip = skipReason(key); + it(`${entry.storyboard}/${entry.step}: ${entry.path} exists in ${entry.task} schema`, { skip }, () => { + const segments = parsePath(entry.path); + const reachable = isPathReachable(schema, segments); + assert.ok( + reachable, + `Path "${entry.path}" is not reachable in ${entry.task} response schema. ` + + `Segments: ${JSON.stringify(segments)}` + ); + }); + } + }); + describe('context extractor tasks have registered response schemas', () => { const extractorTasks = Object.keys(CONTEXT_EXTRACTORS); diff --git a/test/lib/storyboard-runner-contract.test.js b/test/lib/storyboard-runner-contract.test.js index e8d532c78..68e4764ff 100644 --- a/test/lib/storyboard-runner-contract.test.js +++ b/test/lib/storyboard-runner-contract.test.js @@ -71,6 +71,155 @@ describe('runner-output contract: validation_result', () => { assert.strictEqual(result.actual, 'canceled'); }); + describe('field_value_or_absent', () => { + test('passes when the field is absent', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'replayed', + allowed_values: [false], + description: 'replayed is either absent or false on fresh execution', + }, + { taskResult: { success: true, data: {} } } + ); + assert.strictEqual(result.passed, true); + assert.strictEqual(result.json_pointer, '/replayed'); + assert.strictEqual(result.check, 'field_value_or_absent'); + }); + + test('passes when present and in allowed_values', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'replayed', + allowed_values: [false], + description: 'replayed is either absent or false on fresh execution', + }, + { taskResult: { success: true, data: { replayed: false } } } + ); + assert.strictEqual(result.passed, true); + }); + + test('fails when present with a disallowed value', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'replayed', + allowed_values: [false], + description: 'replayed is either absent or false on fresh execution', + }, + { taskResult: { success: true, data: { replayed: true } } } + ); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.json_pointer, '/replayed'); + assert.deepStrictEqual(result.expected, [false]); + assert.strictEqual(result.actual, true); + }); + + test('with scalar value fails only on present-mismatch', () => { + const present = runOne( + { + check: 'field_value_or_absent', + path: 'status', + value: 'active', + description: 'status is either absent or active', + }, + { taskResult: { success: true, data: { status: 'paused' } } } + ); + assert.strictEqual(present.passed, false); + assert.strictEqual(present.expected, 'active'); + assert.strictEqual(present.actual, 'paused'); + + const absent = runOne( + { + check: 'field_value_or_absent', + path: 'status', + value: 'active', + description: 'status is either absent or active', + }, + { taskResult: { success: true, data: {} } } + ); + assert.strictEqual(absent.passed, true); + }); + + test('with a null field fails (null is present, not absent)', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'replayed', + allowed_values: [false], + description: 'replayed is either absent or false', + }, + { taskResult: { success: true, data: { replayed: null } } } + ); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.actual, null); + }); + + // `resolvePath` returns `undefined` whether the key is truly missing or + // present with an explicit `undefined` value (JSON cannot encode this case + // anyway, but native handlers can produce it). The matcher treats both + // the same — absent semantics win. Pinning this disambiguates for any + // implementor walking the runner against a non-JSON transport. + test('treats an explicit `undefined` value the same as a missing key', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'replayed', + allowed_values: [false], + description: 'replayed is either absent or false', + }, + { taskResult: { success: true, data: { replayed: undefined } } } + ); + assert.strictEqual(result.passed, true); + }); + + // A broken chain mid-path (intermediate segment missing) resolves to + // `undefined`, which the matcher routes to the absent-passes branch. + // Same outcome as the leaf being missing — the whole chain "is absent." + test('broken chain in the middle of the path is treated as absent', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'envelope.status.replayed', + allowed_values: [false], + description: 'envelope.status.replayed is either absent or false', + }, + { taskResult: { success: true, data: { envelope: {} } } } + ); + assert.strictEqual(result.passed, true); + }); + + // Empty `allowed_values` falls through to the scalar `value` branch, same + // as `field_value`. With neither set, the matcher compares against + // `undefined`. Storyboards shouldn't emit this shape, but pinning the + // behavior means a silent authoring bug surfaces as a failing check + // rather than silently passing. + test('empty allowed_values falls through to `value` comparison (same as field_value)', () => { + const result = runOne( + { + check: 'field_value_or_absent', + path: 'status', + allowed_values: [], + value: 'active', + description: 'status is either absent or active', + }, + { taskResult: { success: true, data: { status: 'active' } } } + ); + assert.strictEqual(result.passed, true); + }); + + test('missing `path` returns a structured error', () => { + const result = runOne( + { check: 'field_value_or_absent', allowed_values: [false], description: 'no path supplied' }, + { taskResult: { success: true, data: {} } } + ); + assert.strictEqual(result.passed, false); + assert.strictEqual(result.json_pointer, null); + assert.match(result.error, /No path specified for field_value_or_absent/); + }); + }); + test('response_schema failure carries schema_id, schema_url, AJV-shaped actual', () => { const result = runOne( { check: 'response_schema', description: 'Response matches get-adcp-capabilities-response.json schema' },