From 816fcde8d0621b57cab8f6375f3e95c988ee3b20 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Tue, 28 Apr 2026 11:57:26 -0400 Subject: [PATCH 1/2] fix(addie): grade_agent_signing auto-skips capability-profile mismatches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the wrapper started running grades end-to-end (#3424), Addie's first prod demo against /mcp-strict reported "31 pass / 2 fail" — and diagnosed the 2 fails as real verifier bugs. They aren't: vectors 007 and 018 test required-mode and forbidden-mode content-digest enforcement respectively, but /mcp-strict declares covers_content_digest: 'either', so neither vector applies. The in-process gradeRequestSigning() has an `agentCapability` option that auto-skips these as capability_profile_mismatch; the CLI surface doesn't expose it. This wrapper now does the equivalent CLI-side: - Anonymously probes get_adcp_capabilities for request_signing.covers_content_digest. - Maps the declared mode → vectors that don't apply (007 when 'either' or 'forbidden', 018 when 'either' or 'required'). - Passes --skip to the CLI for the matching vectors. - Caller can short-circuit with content_digest_mode for auth-gated routes where the anonymous probe fails. Probe-failure path preserves today's no-skip behavior — better to over-report than silently swallow a real verifier bug. Validated against test-agent.adcontextprotocol.org/mcp-strict: - Without skip: 31 pass / 2 fail / 6 skip (vectors 007, 018 false-fail). - With auto-skip: 31 pass / 0 fail / 8 skip (clean report; probe succeeds when called with content_digest_mode=either). Real long-term fix is upstreaming a capability-aware --auto-skip flag to @adcp/client's CLI; the wrapper drops the hardcoded mapping then. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/grader-capability-auto-skip.md | 6 ++ server/src/addie/mcp/auth-grader-tools.ts | 88 +++++++++++++++++++++ server/tests/unit/auth-grader-tools.test.ts | 33 ++++++++ 3 files changed, 127 insertions(+) create mode 100644 .changeset/grader-capability-auto-skip.md diff --git a/.changeset/grader-capability-auto-skip.md b/.changeset/grader-capability-auto-skip.md new file mode 100644 index 0000000000..8becbaf46e --- /dev/null +++ b/.changeset/grader-capability-auto-skip.md @@ -0,0 +1,6 @@ +--- +--- + +`grade_agent_signing` now auto-skips request-signing vectors whose `verifier_capability.covers_content_digest` doesn't match the agent's declared mode — the CLI-side reimplementation of the grader's in-process `agentCapability` option, which the CLI surface doesn't expose. Default behavior anonymously probes `get_adcp_capabilities` and reads `request_signing.covers_content_digest`; callers can short-circuit with `content_digest_mode: 'either' | 'required' | 'forbidden'` when probing fails (auth-gated routes). + +Validated against `https://test-agent.adcontextprotocol.org/mcp-strict`: report goes from `31 pass / 2 fail / 6 skip` (with vectors 007 and 018 reported as false-failures) to `31 pass / 0 fail / 8 skip`. The two extra skips are correctly classified as capability-profile mismatches. diff --git a/server/src/addie/mcp/auth-grader-tools.ts b/server/src/addie/mcp/auth-grader-tools.ts index fe7edf27a0..3fcf00db45 100644 --- a/server/src/addie/mcp/auth-grader-tools.ts +++ b/server/src/addie/mcp/auth-grader-tools.ts @@ -44,6 +44,72 @@ const ADCP_CLIENT_BIN = (() => { const logger = createLogger('addie-auth-grader-tools'); +type ContentDigestMode = 'either' | 'required' | 'forbidden'; + +/** + * Probe the agent's `request_signing.covers_content_digest` mode via a + * JSON-RPC `tools/call` of `get_adcp_capabilities`. Lets us pre-skip the + * mode-mismatch vectors the grader would otherwise report as failures — + * `agentCapability` does this in-process, but the CLI doesn't expose it. + * + * Returns null (skip nothing) on any probe failure: better to over-report + * than to silently swallow a real verifier bug. + */ +async function probeContentDigestMode(agentUrl: string): Promise { + try { + const res = await fetch(agentUrl, { + method: 'POST', + headers: { 'content-type': 'application/json', accept: 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + method: 'tools/call', + params: { name: 'get_adcp_capabilities', arguments: {} }, + id: 'addie-capability-probe', + }), + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) return null; + const body = await res.json() as { + result?: { content?: Array<{ text?: string }> }; + }; + const text = body.result?.content?.[0]?.text; + if (!text) return null; + const cap = JSON.parse(text) as { + request_signing?: { covers_content_digest?: string }; + }; + const mode = cap.request_signing?.covers_content_digest; + if (mode === 'either' || mode === 'required' || mode === 'forbidden') return mode; + return null; + } catch (err) { + logger.debug( + { err: err instanceof Error ? err.message : String(err), agentUrl }, + 'capability probe failed; grader will run without auto-skip' + ); + return null; + } +} + +/** + * Vectors whose `verifier_capability.covers_content_digest` clashes with + * the declared agent mode. Hardcoded against `@adcp/client`@5.21.x test + * vectors — extend when new content-digest vectors land. The grader's + * in-process `agentCapability` option does this comparison automatically; + * this is the CLI-side reimplementation. + * + * Exported for unit testing. + */ +export function contentDigestSkipsForMode(mode: ContentDigestMode | null): string[] { + if (!mode) return []; + const skip: string[] = []; + // Vector 007 expects a `required` agent to reject a digest-less signature. + // Skip when the agent declares `either` (ambivalent) or `forbidden`. + if (mode === 'either' || mode === 'forbidden') skip.push('007-missing-content-digest'); + // Vector 018 expects a `forbidden` agent to reject a digest-covering signature. + // Skip when the agent declares `either` or `required`. + if (mode === 'either' || mode === 'required') skip.push('018-digest-covered-when-forbidden'); + return skip; +} + function validateAgentUrl(url: string): string | null { let parsed: URL; try { @@ -93,6 +159,11 @@ export const AUTH_GRADER_TOOLS: AddieTool[] = [ enum: ['mcp', 'raw'], description: 'Transport mode. `mcp` (default) wraps each vector body in a JSON-RPC tools/call envelope and posts to the agent\'s MCP mount — right for AdCP MCP servers. `raw` posts to per-operation AdCP endpoints — for agents that expose a raw HTTP surface.', }, + content_digest_mode: { + type: 'string', + enum: ['either', 'required', 'forbidden'], + description: 'The agent\'s declared `request_signing.covers_content_digest` mode. When set, vectors that test the inverse modes auto-skip (mirrors the in-process grader\'s `agentCapability` option, which the CLI doesn\'t expose). Leave unset to probe `get_adcp_capabilities` anonymously; the probe falls back to no-skip on auth-gated routes.', + }, }, required: ['agent_url'], }, @@ -143,11 +214,28 @@ export function createAuthGraderToolHandlers(): Map< // every Addie-grade-able agent today is MCP-style (JSON-RPC tools/call), // and `raw` against an MCP mount returns 404 on every probe. Operators // who genuinely have a raw AdCP endpoint can pass `transport: 'raw'`. + // Pre-flight `get_adcp_capabilities` so we can pre-skip vectors whose + // verifier_capability profile doesn't match what the agent advertises + // (the in-process grader does this via `agentCapability`; the CLI + // doesn't expose that option). The caller can short-circuit via + // `content_digest_mode` — useful when the route requires auth and the + // anonymous probe can't read the capability declaration. + const explicitMode = + input.content_digest_mode === 'either' || + input.content_digest_mode === 'required' || + input.content_digest_mode === 'forbidden' + ? (input.content_digest_mode as ContentDigestMode) + : null; + const probedMode = + explicitMode ?? (rawTransport ? null : await probeContentDigestMode(agentUrl)); + const autoSkip = contentDigestSkipsForMode(probedMode); + const args = [ADCP_CLIENT_BIN, 'grade', 'request-signing', agentUrl, '--json']; args.push('--transport', rawTransport ? 'raw' : 'mcp'); if (allowLive) args.push('--allow-live-side-effects'); else args.push('--skip-rate-abuse'); if (allowHttp) args.push('--allow-http'); + if (autoSkip.length > 0) args.push('--skip', autoSkip.join(',')); try { // 90s is enough for the safe-default path (rate-abuse skipped, ~25 diff --git a/server/tests/unit/auth-grader-tools.test.ts b/server/tests/unit/auth-grader-tools.test.ts index 9b1de87409..0f7845eff2 100644 --- a/server/tests/unit/auth-grader-tools.test.ts +++ b/server/tests/unit/auth-grader-tools.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest'; import { AUTH_GRADER_TOOLS, createAuthGraderToolHandlers, + contentDigestSkipsForMode, } from '../../src/addie/mcp/auth-grader-tools.js'; describe('auth grader tools', () => { @@ -49,3 +50,35 @@ describe('auth grader tools', () => { expect(out).toContain('HTTP or HTTPS'); }); }); + +describe('contentDigestSkipsForMode', () => { + it('returns no skips when mode is null (probe failed)', () => { + // Probe failure must NOT swallow real verifier bugs — better to over-report. + expect(contentDigestSkipsForMode(null)).toEqual([]); + }); + + it("'either' mode skips both 007 and 018", () => { + // Agent declares it accepts signatures with or without content-digest; + // both forced-mode vectors (required + forbidden) are inapplicable. + expect(contentDigestSkipsForMode('either').sort()).toEqual([ + '007-missing-content-digest', + '018-digest-covered-when-forbidden', + ]); + }); + + it("'required' mode skips only 018 (digest-when-forbidden)", () => { + // Agent requires content-digest coverage; the negative vector that + // tests forbidden-mode rejection doesn't apply. + expect(contentDigestSkipsForMode('required')).toEqual([ + '018-digest-covered-when-forbidden', + ]); + }); + + it("'forbidden' mode skips only 007 (missing-content-digest)", () => { + // Agent forbids content-digest coverage; the negative vector that + // tests required-mode rejection doesn't apply. + expect(contentDigestSkipsForMode('forbidden')).toEqual([ + '007-missing-content-digest', + ]); + }); +}); From f8a041979dcfd5b0b282c66b6d2cc9cc83ff1650 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Tue, 28 Apr 2026 12:02:14 -0400 Subject: [PATCH 2/2] fix(addie): SSRF-defend the capability probe in grade_agent_signing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security review on PR #3452 (🟡): the bare fetch() in probeContentDigestMode bypassed the SSRF gate that the grader CLI's ssrfSafeFetch enforces. A caller with an authenticated session could supply a URL whose DNS A record resolves to 127.0.0.1 / 169.254.169.254 / RFC1918 — fetch would hit the loopback/metadata endpoint before the grader CLI's own SSRF gate could refuse, and an error response would land in logger.debug. Plumb the local validateFetchUrl from server/src/utils/url-security.ts which DNS-resolves the hostname and rejects any address in private, link-local, loopback, CGNAT, IPv6 ULA, or always-blocked ranges. Add a 64 KiB body cap matching ssrfSafeFetch's default. Drop the agent URL from the probe-failure debug log to close the info-disclosure side channel. Set redirect: 'manual' so a 30x to a private host can't move the connection laterally. Co-Authored-By: Claude Opus 4.7 (1M context) --- server/src/addie/mcp/auth-grader-tools.ts | 29 ++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/server/src/addie/mcp/auth-grader-tools.ts b/server/src/addie/mcp/auth-grader-tools.ts index 3fcf00db45..4160af3d5b 100644 --- a/server/src/addie/mcp/auth-grader-tools.ts +++ b/server/src/addie/mcp/auth-grader-tools.ts @@ -21,6 +21,7 @@ import { runAuthDiagnosis, type AuthDiagnosisReport } from '@adcp/client/auth'; import type { AddieTool } from '../types.js'; import type { AgentConfig } from '@adcp/client/types'; import { createLogger } from '../../logger.js'; +import { sanitizeUrl, validateFetchUrl } from '../../utils/url-security.js'; const execFileAsync = promisify(execFile); @@ -46,6 +47,11 @@ const logger = createLogger('addie-auth-grader-tools'); type ContentDigestMode = 'either' | 'required' | 'forbidden'; +/** Cap response body size at 64 KiB — capabilities responses are tiny; anything + * larger is either a misbehaving agent or an attempted memory-exhaustion against + * the prod server. Mirrors `@adcp/client`'s `ssrfSafeFetch` default. */ +const PROBE_BODY_CAP_BYTES = 64 * 1024; + /** * Probe the agent's `request_signing.covers_content_digest` mode via a * JSON-RPC `tools/call` of `get_adcp_capabilities`. Lets us pre-skip the @@ -54,10 +60,16 @@ type ContentDigestMode = 'either' | 'required' | 'forbidden'; * * Returns null (skip nothing) on any probe failure: better to over-report * than to silently swallow a real verifier bug. + * + * SSRF defense: `validateFetchUrl` resolves the hostname and rejects any URL + * whose A/AAAA records land on private/link-local/loopback addresses before + * we issue the fetch. Body is capped at PROBE_BODY_CAP_BYTES. */ async function probeContentDigestMode(agentUrl: string): Promise { try { - const res = await fetch(agentUrl, { + const url = new URL(agentUrl); + await validateFetchUrl(url); + const res = await fetch(sanitizeUrl(url), { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json' }, body: JSON.stringify({ @@ -67,14 +79,19 @@ async function probeContentDigestMode(agentUrl: string): Promise PROBE_BODY_CAP_BYTES) return null; + const text = await res.text(); + if (text.length > PROBE_BODY_CAP_BYTES) return null; + const body = JSON.parse(text) as { result?: { content?: Array<{ text?: string }> }; }; - const text = body.result?.content?.[0]?.text; - if (!text) return null; - const cap = JSON.parse(text) as { + const inner = body.result?.content?.[0]?.text; + if (!inner) return null; + const cap = JSON.parse(inner) as { request_signing?: { covers_content_digest?: string }; }; const mode = cap.request_signing?.covers_content_digest; @@ -82,7 +99,7 @@ async function probeContentDigestMode(agentUrl: string): Promise