From 329b08240d06cbb24bc45f5c02c15748ec4b5891 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 25 Apr 2026 05:58:52 -0400 Subject: [PATCH 1/2] docs(conformance, catalog): refresh universal-storyboards tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both docs/building/conformance.mdx and docs/building/compliance-catalog.mdx had stale universal-storyboards tables. static/compliance/source/universal/ now contains 9 graded storyboards; the docs listed 5–7. Drift accumulated as new storyboards landed without back-filling the index pages. - Add webhook-emission (universal since #2417 / 3.0; never indexed) - Add pagination-integrity (recently landed; missing from both) - Add idempotency to compliance-catalog (had it in conformance.mdx only) - Add signed-requests to compliance-catalog (had it in conformance.mdx only, added there in #3077) Framing fix in compliance-catalog: the lead-in claimed every agent runs every universal storyboard regardless of claims — true in scope, but confusing on capability-gated rows. Reworded to acknowledge the gating (deterministic-testing, signed-requests) plus a closing paragraph that gated storyboards can't be partially implemented (advertise false rather than ship a partial surface). Both index pages now reflect the same 9 graded universal storyboards in the same order, matching the published /compliance/{version}/universal/ directory. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../refresh-universal-storyboard-tables.md | 20 +++++++++++++++++++ docs/building/compliance-catalog.mdx | 16 ++++++++++----- docs/building/conformance.mdx | 2 ++ 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 .changeset/refresh-universal-storyboard-tables.md diff --git a/.changeset/refresh-universal-storyboard-tables.md b/.changeset/refresh-universal-storyboard-tables.md new file mode 100644 index 0000000000..1b00df2f94 --- /dev/null +++ b/.changeset/refresh-universal-storyboard-tables.md @@ -0,0 +1,20 @@ +--- +--- + +docs(conformance, catalog): refresh universal-storyboards tables to match `static/compliance/source/universal/` + +Both `docs/building/conformance.mdx` and `docs/building/compliance-catalog.mdx` had stale universal-storyboards tables. The actual `static/compliance/source/universal/` directory now contains 9 graded storyboards; the docs listed 5–7. Drift accumulated as new storyboards landed without back-filling the index pages. + +**Adds (both files):** + +- `webhook-emission` — outbound webhook conformance (idempotency_key + RFC 9421 webhook signing). Runs for any agent accepting `push_notification_config`. Has been universal since #2417 / 3.0; never indexed in the catalog tables. +- `pagination-integrity` — `cursor` ↔ `has_more` invariant. Recently landed; missing from both pages. + +**Adds (compliance-catalog only):** + +- `idempotency` — was missing entirely from the catalog table though present in conformance.mdx and shipped as universal in 3.0. +- `signed-requests` — added in #3077 to the conformance.mdx table; the parallel catalog entry was missed. + +**Framing fix (compliance-catalog):** the lead-in said "Every agent runs every storyboard… regardless of which protocols or specialisms it claims," which is true in scope but confusing on capability-gated rows (`deterministic-testing`, `signed-requests`). Reworded to "every agent runs every storyboard, with a few capability-gated by an explicit `supported: true` advertisement" plus a closing paragraph noting that gated storyboards can't be partially implemented (advertise `false` if you don't ship the full surface). + +No new normative content. The two index pages now reflect the same 9 universal storyboards in the same order, matching the published `/compliance/{version}/universal/` directory. diff --git a/docs/building/compliance-catalog.mdx b/docs/building/compliance-catalog.mdx index fa36e29bb4..f6a022408c 100644 --- a/docs/building/compliance-catalog.mdx +++ b/docs/building/compliance-catalog.mdx @@ -15,15 +15,21 @@ This page is the human-readable index of that taxonomy. The machine-readable equ ## Universal storyboards -Every agent runs every storyboard in `/compliance/{version}/universal/` regardless of which protocols or specialisms it claims. Failing a universal storyboard fails overall compliance. +Every agent runs every storyboard in `/compliance/{version}/universal/` regardless of which protocols or specialisms it claims. A few are *capability-gated* — they only run when the agent advertises the relevant capability — but the storyboard is still universal in scope: any agent claiming the capability is graded by it. Failing a universal storyboard fails overall compliance. | Storyboard | Purpose | |-----------|---------| -| `capability-discovery` | `get_adcp_capabilities` shape and version declaration | -| `schema-validation` | Response schema conformance and temporal constraints | -| `error-compliance` | Structured errors with correct codes and transport binding | -| `deterministic-testing` | `comply_test_controller` state-machine verification (skipped if unsupported) | +| `capability-discovery` | `get_adcp_capabilities` shape, protocol/specialism declarations, version advertising | +| `schema-validation` | Request and response schema conformance, ISO 8601 timestamps, temporal invariants | +| `error-compliance` | Structured error shape, published error codes, transport binding, no existence leaks across tenants | +| `idempotency` | `idempotency_key` scoping, replay semantics, `IDEMPOTENCY_CONFLICT`, `replayed: true`, declared TTL | | `security` | **Authentication baseline — unauth rejection, API key enforcement, OAuth discovery + RFC 9728 audience binding.** See [Authentication](/docs/building/integration/authentication). | +| `webhook-emission` | Outbound webhook conformance — stable `idempotency_key` across retries; RFC 9421 webhook signing (or HMAC fallback if the buyer opted in). Runs for any agent that accepts `push_notification_config` on any operation. | +| `pagination-integrity` | `cursor` ↔ `has_more` invariant verified by walking a paginated `list_creatives` response from a continuation page through to terminal. | +| `deterministic-testing` | `comply_test_controller` state-machine verification — skipped if `capabilities.compliance_testing.supported: false`. | +| `signed-requests` | RFC 9421 transport-layer request-signing verification — skipped if `request_signing.supported: false`. | + +Capability-gated rows (`deterministic-testing`, `signed-requests`) are skipped only when the agent advertises the capability as `false`; they cannot be claimed and partially implemented. Declaring `supported: true` and failing the storyboard is non-conformant — declare `false` rather than ship a partial implementation. ## Protocols diff --git a/docs/building/conformance.mdx b/docs/building/conformance.mdx index 23f0975e75..9c5c10692a 100644 --- a/docs/building/conformance.mdx +++ b/docs/building/conformance.mdx @@ -55,6 +55,8 @@ Every agent MUST pass every storyboard below. | [`error_compliance`](https://adcontextprotocol.org/compliance/latest/universal/error-compliance) | Structured error shape, published error codes, transport binding, no existence leaks across tenants | | [`idempotency`](https://adcontextprotocol.org/compliance/latest/universal/idempotency) | `idempotency_key` scoping, replay semantics, `IDEMPOTENCY_CONFLICT`, `replayed: true`, declared TTL | | [`security_baseline`](https://adcontextprotocol.org/compliance/latest/universal/security) | Unauth rejection, API key enforcement, OAuth discovery + RFC 9728 audience binding | +| [`webhook_emission`](https://adcontextprotocol.org/compliance/latest/universal/webhook-emission) | Outbound webhook conformance — stable `idempotency_key` across retries, RFC 9421 webhook signing (or opt-in HMAC fallback) on every delivery. Runs for any agent accepting `push_notification_config`. | +| [`pagination_integrity`](https://adcontextprotocol.org/compliance/latest/universal/pagination-integrity) | `cursor` ↔ `has_more` invariant on paginated `list_creatives` responses, walked from a continuation page through to a terminal page | | [`deterministic_testing`](https://adcontextprotocol.org/compliance/latest/universal/deterministic-testing) | `comply_test_controller` state machine — skipped if `capabilities.compliance_testing.supported: false` | | [`signed_requests`](https://adcontextprotocol.org/compliance/latest/universal/signed-requests) | RFC 9421 transport-layer request-signing verification — skipped if `request_signing.supported: false`. | From 8a8ca112fa0e8a79a4de634cf6d8f747b0a9ce7c Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 25 Apr 2026 07:46:20 -0400 Subject: [PATCH 2/2] chore(compliance): lint universal-storyboard doc tables match filesystem (#3102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(compliance): lint that universal-storyboard doc tables match the filesystem Prevents the drift #3099 just fixed from re-accumulating. Every graded universal storyboard MUST appear in both docs/building/conformance.mdx and docs/building/compliance-catalog.mdx; every backtick-quoted slug in those tables MUST resolve to a real graded storyboard on disk. Catches: - New universal storyboard ships without a doc-table row (forward). - Doc keeps a row for a renamed/deleted storyboard (reverse). - Either index page loses its "Universal" heading. Wiring: - scripts/lint-universal-storyboard-doc-parity.cjs — new module exporting lint({ sourceDir, repoRoot }) plus helpers, with a CLI entrypoint. - scripts/build-compliance.cjs — calls lint inside generateIndex, so the compliance build fails loudly on drift. - tests/lint-universal-storyboard-doc-parity.test.cjs — 10 tests: source-tree guard, clean fixture, non-graded fixtures filtered out, forward parity (both docs), reverse parity (both docs), missing heading, helper unit tests. - package.json — test:storyboard-doc-parity wired into the umbrella test target alongside other storyboard lints; CI picks it up automatically via npm run test. "Graded" = YAML has a phases:[] array. Filters out the three non-graded fixtures (storyboard-schema.yaml, runner-output-contract.yaml, fictional-entities.yaml) that live alongside graded storyboards. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(compliance): apply DX-review nice-to-haves to doc-parity lint Two cheap improvements from the DX-expert review: 1. Forward-parity error message now mentions the slug form (kebab-case filename slug for compliance-catalog, snake_case YAML id for conformance). Parallels the reverse error which already had this hint. Helps a contributor adding a brand-new storyboard pick the right form without comparing existing rows. 2. JSX comment above each universal-storyboards table pointing at the lint script. Discoverability for contributors editing the docs directly — they see the rule before the build fails. Tests still pass 10/10; build clean. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../lint-universal-storyboard-doc-parity.md | 23 ++ docs/building/compliance-catalog.mdx | 2 + docs/building/conformance.mdx | 2 + package.json | 3 +- scripts/build-compliance.cjs | 6 + .../lint-universal-storyboard-doc-parity.cjs | 165 ++++++++++++++ ...t-universal-storyboard-doc-parity.test.cjs | 208 ++++++++++++++++++ 7 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 .changeset/lint-universal-storyboard-doc-parity.md create mode 100644 scripts/lint-universal-storyboard-doc-parity.cjs create mode 100644 tests/lint-universal-storyboard-doc-parity.test.cjs diff --git a/.changeset/lint-universal-storyboard-doc-parity.md b/.changeset/lint-universal-storyboard-doc-parity.md new file mode 100644 index 0000000000..765dff567e --- /dev/null +++ b/.changeset/lint-universal-storyboard-doc-parity.md @@ -0,0 +1,23 @@ +--- +--- + +chore(compliance): lint that universal-storyboard doc tables match the filesystem + +New build-time + unit-test lint that prevents the drift #3099 just fixed from re-accumulating. Every graded universal storyboard MUST appear in both `docs/building/conformance.mdx` and `docs/building/compliance-catalog.mdx`; every backtick-quoted slug in those tables MUST resolve to a real graded storyboard on disk. + +**Why patch / `--empty` changeset.** Build-script + tests + new lint logic — no protocol-spec surface change. The lint encodes existing convention (the two index pages should agree with `static/compliance/source/universal/`), it doesn't introduce a new normative rule. + +**What it catches:** + +- New universal storyboard ships without a doc-table row → forward parity fails (specific "missing rows for X" error). +- Doc keeps a row for a renamed/deleted storyboard → reverse parity fails ("references X but no graded storyboard exists" error). +- Either index page loses its "Universal" heading → "missing expected heading" error. + +**Wiring:** + +- `scripts/lint-universal-storyboard-doc-parity.cjs` — new module exporting `lint({ sourceDir, repoRoot })` plus helpers, with a CLI entrypoint. +- `scripts/build-compliance.cjs` — calls the lint inside `generateIndex`, between `verifyEnumParity` and `lintStoryboardIdempotency`. Build fails loudly if drift is present. +- `tests/lint-universal-storyboard-doc-parity.test.cjs` — 10 tests: source-tree guard, clean-fixture, non-graded-fixture filtering, forward parity (both docs), reverse parity (both docs), missing-heading, helper unit tests. +- `package.json` — new `test:storyboard-doc-parity` script wired into the umbrella `test` target alongside the other storyboard lints, so CI's existing `npm run test` invocation picks it up automatically. + +Identifies "graded" by the presence of a `phases:` array in the YAML. Filters out the three non-graded fixtures (`storyboard-schema.yaml`, `runner-output-contract.yaml`, `fictional-entities.yaml`) which live alongside graded storyboards but aren't run by the suite. diff --git a/docs/building/compliance-catalog.mdx b/docs/building/compliance-catalog.mdx index f6a022408c..d869f1cd8c 100644 --- a/docs/building/compliance-catalog.mdx +++ b/docs/building/compliance-catalog.mdx @@ -17,6 +17,8 @@ This page is the human-readable index of that taxonomy. The machine-readable equ Every agent runs every storyboard in `/compliance/{version}/universal/` regardless of which protocols or specialisms it claims. A few are *capability-gated* — they only run when the agent advertises the relevant capability — but the storyboard is still universal in scope: any agent claiming the capability is graded by it. Failing a universal storyboard fails overall compliance. +{/* Lint: scripts/lint-universal-storyboard-doc-parity.cjs keeps this table in sync with static/compliance/source/universal/. Add a row (kebab-case slug) when you add a graded storyboard; remove the row when one is deleted. The build fails on drift. */} + | Storyboard | Purpose | |-----------|---------| | `capability-discovery` | `get_adcp_capabilities` shape, protocol/specialism declarations, version advertising | diff --git a/docs/building/conformance.mdx b/docs/building/conformance.mdx index 9c5c10692a..a7fb81f2cb 100644 --- a/docs/building/conformance.mdx +++ b/docs/building/conformance.mdx @@ -48,6 +48,8 @@ Agents MUST NOT declare a capability whose storyboards they do not pass. See the Every agent MUST pass every storyboard below. +{/* Lint: scripts/lint-universal-storyboard-doc-parity.cjs keeps this table in sync with static/compliance/source/universal/. Add a row (snake_case YAML id) when you add a graded storyboard; remove the row when one is deleted. The build fails on drift. */} + | Storyboard | What it verifies | |------------|------------------| | [`capability_discovery`](https://adcontextprotocol.org/compliance/latest/universal/capability-discovery) | `get_adcp_capabilities` shape, protocol/specialism declarations, version advertising | diff --git a/package.json b/package.json index ea1a78d208..4727a7d6ed 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:storyboard-auth-shape": "node --test tests/lint-storyboard-auth-shape.test.cjs", "test:storyboard-test-kits": "node --test tests/lint-storyboard-test-kits.test.cjs", "test:storyboard-sample-request-schema": "node --test tests/lint-storyboard-sample-request-schema.test.cjs", + "test:storyboard-doc-parity": "node --test tests/lint-universal-storyboard-doc-parity.test.cjs", "test:pagination-invariant": "node --test tests/lint-pagination-invariant.test.cjs", "test:error-codes": "node scripts/lint-error-codes.cjs", "test:substitution-vector-names": "node scripts/lint-substitution-vector-names.cjs", @@ -50,7 +51,7 @@ "test:docker": "docker compose -f docker-compose.test.yml up --build --abort-on-container-exit", "test:docs-nav": "node tests/docs-nav-validation.test.cjs", "test:platform-agnostic": "node tests/check-platform-agnostic.cjs", - "test": "npm run test:docs-nav && npm run test:schemas && npm run test:examples && npm run test:extensions && npm run test:extension-schemas && npm run test:error-handling && npm run test:json-schema && npm run test:composed && npm run test:migrations && npm run test:hmac-vectors && npm run test:hmac-signer-conformance && npm run test:transport-errors && npm run test:targeting-overlay-vectors && npm run test:storyboard-scoping && npm run test:storyboard-branch-sets && npm run test:storyboard-contradictions && npm run test:storyboard-context-entity && npm run test:storyboard-auth-shape && npm run test:storyboard-test-kits && npm run test:storyboard-sample-request-schema && npm run test:pagination-invariant && npm run test:error-codes && npm run test:substitution-vector-names && npm run test:platform-agnostic && npm run test:unit && npm run test:server-unit && npm run test:openapi && npm run typecheck", + "test": "npm run test:docs-nav && npm run test:schemas && npm run test:examples && npm run test:extensions && npm run test:extension-schemas && npm run test:error-handling && npm run test:json-schema && npm run test:composed && npm run test:migrations && npm run test:hmac-vectors && npm run test:hmac-signer-conformance && npm run test:transport-errors && npm run test:targeting-overlay-vectors && npm run test:storyboard-scoping && npm run test:storyboard-branch-sets && npm run test:storyboard-contradictions && npm run test:storyboard-context-entity && npm run test:storyboard-auth-shape && npm run test:storyboard-test-kits && npm run test:storyboard-sample-request-schema && npm run test:storyboard-doc-parity && npm run test:pagination-invariant && npm run test:error-codes && npm run test:substitution-vector-names && npm run test:platform-agnostic && npm run test:unit && npm run test:server-unit && npm run test:openapi && npm run typecheck", "test:all": "npm run test:schemas && npm run test:examples && npm run test:extensions && npm run test:error-handling && npm run test:snippets && npm run typecheck", "precommit": "bash scripts/with-timeout.sh 60 npm run test:unit && npm run typecheck", "prepare": "husky", diff --git a/scripts/build-compliance.cjs b/scripts/build-compliance.cjs index 3821e33ada..e2db1c8b98 100644 --- a/scripts/build-compliance.cjs +++ b/scripts/build-compliance.cjs @@ -345,10 +345,16 @@ function verifyEnumParity(specialisms, protocols) { } } +const { lint: lintUniversalDocParity } = require('./lint-universal-storyboard-doc-parity.cjs'); + function generateIndex(version, sourceDir) { const specialisms = discoverSpecialisms(sourceDir); const protocols = discoverProtocols(sourceDir, specialisms); verifyEnumParity(specialisms, protocols); + const docParityErrors = lintUniversalDocParity({ sourceDir }); + if (docParityErrors.length) { + throw new Error('Universal-storyboard doc parity drift:\n - ' + docParityErrors.join('\n - ')); + } lintStoryboardIdempotency(sourceDir, SCHEMAS_DIR); const universalDir = path.join(sourceDir, 'universal'); const universal = fs.existsSync(universalDir) diff --git a/scripts/lint-universal-storyboard-doc-parity.cjs b/scripts/lint-universal-storyboard-doc-parity.cjs new file mode 100644 index 0000000000..ef2aed56d0 --- /dev/null +++ b/scripts/lint-universal-storyboard-doc-parity.cjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node +/** + * Lint: docs index pages list every graded universal storyboard, and only + * reference storyboards that exist on disk. + * + * Why + * --- + * The catalog and conformance index pages each carry a table of universal + * storyboards. Both have drifted in the past — storyboards landed without + * back-filling the docs (#3099 caught webhook-emission, pagination-integrity, + * idempotency, signed-requests all missing in various places). This lint + * keeps them honest. + * + * Forward parity: every graded universal storyboard MUST appear in both + * tables. A new universal storyboard that ships without a row breaks the + * build. + * + * Reverse parity: every backtick-quoted slug appearing in a table row MUST + * resolve to a real graded storyboard on disk. A copy-paste typo or a row + * left behind after a rename breaks the build. + * + * "Graded" means the YAML has a `phases:` array. Filters out non-graded + * fixtures (storyboard-schema.yaml, runner-output-contract.yaml, + * fictional-entities.yaml) which live alongside graded storyboards but + * aren't run by the suite. + * + * Slug forms differ across the two docs by design (existing convention, + * not changed here): + * - docs/building/conformance.mdx uses snake_case YAML `id` values in markdown links + * - docs/building/compliance-catalog.mdx uses kebab-case filename slugs in plain backticks + */ + +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const yaml = require('js-yaml'); + +const REPO_ROOT = path.join(__dirname, '..'); +const DEFAULT_SOURCE_DIR = path.join(REPO_ROOT, 'static/compliance/source'); + +const DOC_CHECKS = [ + { + relpath: 'docs/building/conformance.mdx', + heading: '## Universal conformance', + tokenForItem: item => item.id, + tokenLabel: 'YAML `id`', + }, + { + relpath: 'docs/building/compliance-catalog.mdx', + heading: '## Universal storyboards', + tokenForItem: item => item.slug, + tokenLabel: 'filename slug', + }, +]; + +function discoverGradedUniversal(sourceDir) { + const universalDir = path.join(sourceDir, 'universal'); + if (!fs.existsSync(universalDir)) return []; + return fs.readdirSync(universalDir) + .filter(f => f.endsWith('.yaml')) + .map(f => { + const slug = f.replace(/\.yaml$/, ''); + let doc = null; + try { + doc = yaml.load(fs.readFileSync(path.join(universalDir, f), 'utf8')); + } catch { + // Malformed YAML is caught by other lints; skip it here so doc parity + // doesn't double-error on the same file. + } + const graded = doc && typeof doc === 'object' && Array.isArray(doc.phases); + const id = doc && typeof doc === 'object' && typeof doc.id === 'string' + ? doc.id + : slug.replace(/-/g, '_'); + return { slug, id, graded }; + }) + .filter(item => item.graded); +} + +function extractSection(content, heading) { + const start = content.indexOf(heading); + if (start === -1) return null; + const tail = content.slice(start + heading.length); + const nextHeadingIdx = tail.search(/\n## /); + return nextHeadingIdx === -1 + ? content.slice(start) + : content.slice(start, start + heading.length + nextHeadingIdx); +} + +function extractTableTokens(section) { + const tokens = new Set(); + for (const line of section.split('\n')) { + if (!line.startsWith('|')) continue; + // First cell only; supports `[`token`](url)` (markdown link) and `` `token` `` (plain). + const m = line.match(/^\|\s*\[?`([a-z][a-z0-9_-]+)`/); + if (m) tokens.add(m[1]); + } + return tokens; +} + +/** + * Run the lint. Returns an array of error strings (empty = clean). + * Pass `{ sourceDir, repoRoot }` to override default paths (used by tests). + */ +function lint({ sourceDir = DEFAULT_SOURCE_DIR, repoRoot = REPO_ROOT } = {}) { + const items = discoverGradedUniversal(sourceDir); + const errors = []; + + for (const check of DOC_CHECKS) { + const filePath = path.join(repoRoot, check.relpath); + if (!fs.existsSync(filePath)) continue; + const content = fs.readFileSync(filePath, 'utf8'); + + const section = extractSection(content, check.heading); + if (section === null) { + errors.push(`${check.relpath}: missing expected heading "${check.heading}"`); + continue; + } + + // Forward parity + const missingFromDoc = items + .map(item => check.tokenForItem(item)) + .filter(token => !section.includes('`' + token + '`')); + if (missingFromDoc.length) { + errors.push( + `${check.relpath}: universal-storyboards table is missing rows for ${missingFromDoc.map(t => '`' + t + '`').join(', ')}.\n` + + ` Add a row to the table under "${check.heading}" using the ${check.tokenLabel} form. The runner will fail any agent that doesn't pass these.` + ); + } + + // Reverse parity + const knownTokens = new Set(items.map(item => check.tokenForItem(item))); + const tableTokens = extractTableTokens(section); + const ghostTokens = [...tableTokens].filter(t => !knownTokens.has(t)); + if (ghostTokens.length) { + errors.push( + `${check.relpath}: universal-storyboards table references ${ghostTokens.map(t => '`' + t + '`').join(', ')} but no graded storyboard exists on disk under static/compliance/source/universal/.\n` + + ` Either add the storyboard YAML or remove the row. (Expected ${check.tokenLabel} form.)` + ); + } + } + + return errors; +} + +function main() { + const errors = lint(); + if (errors.length) { + console.error('Universal-storyboard doc parity drift:\n - ' + errors.join('\n - ')); + process.exit(1); + } + console.log('✓ universal-storyboard doc parity: docs match graded universal storyboards'); +} + +if (require.main === module) { + main(); +} + +module.exports = { + lint, + discoverGradedUniversal, + extractSection, + extractTableTokens, + DOC_CHECKS, +}; diff --git a/tests/lint-universal-storyboard-doc-parity.test.cjs b/tests/lint-universal-storyboard-doc-parity.test.cjs new file mode 100644 index 0000000000..44e8775892 --- /dev/null +++ b/tests/lint-universal-storyboard-doc-parity.test.cjs @@ -0,0 +1,208 @@ +#!/usr/bin/env node +/** + * Tests for the universal-storyboard doc-parity lint. + * + * Three concerns: + * 1. Source-tree guard — the real source + docs pass cleanly. Catches drift + * the day a new universal storyboard lands without a doc-table row. + * 2. Forward parity — synthetic source has a graded storyboard the synthetic + * docs don't list → lint fires with a clear "missing rows for X" error. + * 3. Reverse parity — synthetic docs reference a slug the synthetic source + * doesn't define → lint fires with "no graded storyboard exists" error. + * + * Synthetic fixtures use a temp-dir layout that mirrors the real one: + * /source/universal/{slug}.yaml + * /repo/docs/building/conformance.mdx + * /repo/docs/building/compliance-catalog.mdx + */ + +'use strict'; + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { lint, discoverGradedUniversal, extractTableTokens } = + require('../scripts/lint-universal-storyboard-doc-parity.cjs'); + +test('source tree passes the doc-parity lint', () => { + const errors = lint(); + assert.deepEqual( + errors, + [], + 'real docs drift from real universal storyboards:\n ' + errors.join('\n '), + ); +}); + +function makeFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lint-doc-parity-')); + const sourceDir = path.join(root, 'source'); + const repoRoot = path.join(root, 'repo'); + fs.mkdirSync(path.join(sourceDir, 'universal'), { recursive: true }); + fs.mkdirSync(path.join(repoRoot, 'docs/building'), { recursive: true }); + return { root, sourceDir, repoRoot }; +} + +function writeStoryboard(sourceDir, slug, { graded = true, id = null } = {}) { + const body = graded + ? `id: ${id || slug.replace(/-/g, '_')}\nphases:\n - id: phase_one\n title: "Phase one"\n` + : `# Non-graded fixture\ndescription: not a real storyboard\n`; + fs.writeFileSync(path.join(sourceDir, 'universal', `${slug}.yaml`), body); +} + +function writeConformance(repoRoot, rows) { + const table = rows + .map(([slug, purpose]) => `| [\`${slug}\`](https://example.com/${slug}) | ${purpose} |`) + .join('\n'); + fs.writeFileSync( + path.join(repoRoot, 'docs/building/conformance.mdx'), + `# Conformance\n\n## Universal conformance\n\n| Storyboard | What |\n|------------|------|\n${table}\n\n## Next section\n\nUnrelated content.\n`, + ); +} + +function writeCatalog(repoRoot, rows) { + const table = rows + .map(([slug, purpose]) => `| \`${slug}\` | ${purpose} |`) + .join('\n'); + fs.writeFileSync( + path.join(repoRoot, 'docs/building/compliance-catalog.mdx'), + `# Catalog\n\n## Universal storyboards\n\n| Storyboard | Purpose |\n|-----------|---------|\n${table}\n\n## Protocols\n\nUnrelated.\n`, + ); +} + +test('clean fixture: graded storyboard listed in both docs → no errors', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + writeConformance(repoRoot, [['capability_discovery', 'shape']]); + writeCatalog(repoRoot, [['capability-discovery', 'shape']]); + + const errors = lint({ sourceDir, repoRoot }); + assert.deepEqual(errors, []); +}); + +test('non-graded fixtures (storyboard-schema, runner-output-contract, fictional-entities) are not required in docs', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + // These three live in the real universal/ directory but aren't graded. + writeStoryboard(sourceDir, 'storyboard-schema', { graded: false }); + writeStoryboard(sourceDir, 'runner-output-contract', { graded: false }); + writeStoryboard(sourceDir, 'fictional-entities', { graded: false }); + writeConformance(repoRoot, [['capability_discovery', 'shape']]); + writeCatalog(repoRoot, [['capability-discovery', 'shape']]); + + const errors = lint({ sourceDir, repoRoot }); + assert.deepEqual(errors, []); +}); + +test('forward parity: graded storyboard missing from conformance.mdx → error', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + writeStoryboard(sourceDir, 'webhook-emission'); + // conformance lists only one — webhook_emission is missing. + writeConformance(repoRoot, [['capability_discovery', 'shape']]); + writeCatalog(repoRoot, [ + ['capability-discovery', 'shape'], + ['webhook-emission', 'webhooks'], + ]); + + const errors = lint({ sourceDir, repoRoot }); + assert.equal(errors.length, 1); + assert.match(errors[0], /conformance\.mdx/); + assert.match(errors[0], /missing rows for `webhook_emission`/); +}); + +test('forward parity: graded storyboard missing from compliance-catalog.mdx → error (kebab-case form)', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'pagination-integrity'); + writeConformance(repoRoot, [['pagination_integrity', 'pagination']]); + // catalog forgets the row. + writeCatalog(repoRoot, []); + + const errors = lint({ sourceDir, repoRoot }); + assert.equal(errors.length, 1); + assert.match(errors[0], /compliance-catalog\.mdx/); + assert.match(errors[0], /missing rows for `pagination-integrity`/); +}); + +test('reverse parity: doc references a slug with no graded storyboard on disk → error', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + writeConformance(repoRoot, [ + ['capability_discovery', 'shape'], + ['ghost_storyboard', 'this row references nothing'], + ]); + writeCatalog(repoRoot, [['capability-discovery', 'shape']]); + + const errors = lint({ sourceDir, repoRoot }); + assert.equal(errors.length, 1); + assert.match(errors[0], /conformance\.mdx/); + assert.match(errors[0], /references `ghost_storyboard`/); + assert.match(errors[0], /no graded storyboard exists/); +}); + +test('reverse parity: catalog row for a deprecated/removed storyboard → error', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + writeConformance(repoRoot, [['capability_discovery', 'shape']]); + // catalog still has a row for an old storyboard the YAML was deleted. + writeCatalog(repoRoot, [ + ['capability-discovery', 'shape'], + ['ancient-storyboard', 'should have been deleted'], + ]); + + const errors = lint({ sourceDir, repoRoot }); + assert.equal(errors.length, 1); + assert.match(errors[0], /compliance-catalog\.mdx/); + assert.match(errors[0], /references `ancient-storyboard`/); +}); + +test('missing expected heading produces a specific error', () => { + const { sourceDir, repoRoot } = makeFixture(); + writeStoryboard(sourceDir, 'capability-discovery'); + // Catalog file exists but lacks the expected heading. + fs.writeFileSync( + path.join(repoRoot, 'docs/building/compliance-catalog.mdx'), + '# Catalog\n\nNo universal section here yet.\n', + ); + writeConformance(repoRoot, [['capability_discovery', 'shape']]); + + const errors = lint({ sourceDir, repoRoot }); + assert.equal(errors.length, 1); + assert.match(errors[0], /missing expected heading "## Universal storyboards"/); +}); + +test('discoverGradedUniversal: filters to entries with phases:[]', () => { + const { sourceDir } = makeFixture(); + writeStoryboard(sourceDir, 'graded-one'); + writeStoryboard(sourceDir, 'graded-two', { id: 'custom_id_two' }); + writeStoryboard(sourceDir, 'not-graded', { graded: false }); + + const items = discoverGradedUniversal(sourceDir); + assert.equal(items.length, 2); + const slugs = items.map(i => i.slug).sort(); + assert.deepEqual(slugs, ['graded-one', 'graded-two']); + const idForTwo = items.find(i => i.slug === 'graded-two').id; + assert.equal(idForTwo, 'custom_id_two'); +}); + +test('extractTableTokens: pulls slug from the first cell of each row, ignores prose', () => { + const section = `## Heading + +Some prose mentioning \`unrelated\` in passing. + +| Storyboard | What | +|------------|------| +| [\`capability_discovery\`](https://example.com) | shape | +| \`signed-requests\` | transport | +| [\`pagination_integrity\`](url) | cursor invariant | + +More prose with \`other_token\` that should be ignored. +`; + const tokens = extractTableTokens(section); + assert.deepEqual( + [...tokens].sort(), + ['capability_discovery', 'pagination_integrity', 'signed-requests'], + ); +});