From ec12caab55cb492e825ff355bb529e0c1b4a8cb0 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 14:02:17 +0300 Subject: [PATCH 01/15] feat: add bounded SSE transport observations --- src/core/bounded-http-client.ts | 8 +++- src/core/sse-observation.ts | 77 +++++++++++++++++++++++++++++++ tests/bounded-http-client.test.ts | 38 +++++++++++++++ tests/sse-observation.test.ts | 55 ++++++++++++++++++++++ 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 src/core/sse-observation.ts create mode 100644 tests/sse-observation.test.ts diff --git a/src/core/bounded-http-client.ts b/src/core/bounded-http-client.ts index df6114b..5555b77 100644 --- a/src/core/bounded-http-client.ts +++ b/src/core/bounded-http-client.ts @@ -9,6 +9,7 @@ const DEFAULT_MAX_RESPONSE_BYTES = 1_024 * 1_024; const allowedRequestHeaders = new Set([ "accept", "content-type", + "last-event-id", "mcp-protocol-version", "mcp-session-id", "user-agent" @@ -199,7 +200,12 @@ export async function requestBoundedHttp( let response: http.IncomingMessage | undefined; let settled = false; const timeout = setTimeout(() => { - fail(new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote HTTP request timed out.")); + fail(new BoundedHttpError( + "REMOTE_HTTP_TIMEOUT", + "Remote HTTP request timed out.", + response?.statusCode, + response ? selectSafeHeaders(response.headers) : undefined + )); }, remainingMs); const fail = (error: Error): void => { diff --git a/src/core/sse-observation.ts b/src/core/sse-observation.ts new file mode 100644 index 0000000..a69d756 --- /dev/null +++ b/src/core/sse-observation.ts @@ -0,0 +1,77 @@ +const MAX_SSE_FIELD_BYTES = 1_024; + +export interface SseObservation { + complete: boolean; + eventId: string | null; + retryMs: number | null; + malformed: boolean; +} + +function isSafeEventId(value: Buffer): boolean { + return value.length > 0 + && value.length <= MAX_SSE_FIELD_BYTES + && value.every((byte) => byte >= 0x21 && byte <= 0x7e); +} + +function parseRetry(value: Buffer): number | null { + if (value.length === 0 || value.length > MAX_SSE_FIELD_BYTES) { + return null; + } + + for (const byte of value) { + if (byte < 0x30 || byte > 0x39) { + return null; + } + } + + const retryMs = Number(value.toString("ascii")); + return Number.isSafeInteger(retryMs) ? retryMs : null; +} + +export function observeFirstSseEvent(body: Buffer): SseObservation { + let eventId: string | null = null; + let retryMs: number | null = null; + let malformed = false; + let lineStart = 0; + + for (let index = 0; index < body.length; index += 1) { + if (body[index] !== 0x0a) { + continue; + } + + const lineEnd = index > lineStart && body[index - 1] === 0x0d ? index - 1 : index; + const line = body.subarray(lineStart, lineEnd); + lineStart = index + 1; + + if (line.length === 0) { + return { + complete: true, + eventId: malformed ? null : eventId, + retryMs: malformed ? null : retryMs, + malformed + }; + } + if (line[0] === 0x3a) { + continue; + } + + const separator = line.indexOf(0x3a); + const field = separator === -1 ? line : line.subarray(0, separator); + const valueStart = separator === -1 ? line.length : separator + 1; + const value = line.subarray(valueStart + (line[valueStart] === 0x20 ? 1 : 0)); + + if (field.equals(Buffer.from("id"))) { + if (value.length === 0) { + eventId = null; + } else if (isSafeEventId(value)) { + eventId = value.toString("ascii"); + } else { + malformed = true; + } + } else if (field.equals(Buffer.from("retry"))) { + retryMs = parseRetry(value); + } + } + + return { complete: false, eventId: null, retryMs: null, malformed: false }; +} diff --git a/tests/bounded-http-client.test.ts b/tests/bounded-http-client.test.ts index c0cde8d..49e2ec8 100644 --- a/tests/bounded-http-client.test.ts +++ b/tests/bounded-http-client.test.ts @@ -128,6 +128,18 @@ describe("requestBoundedHttp", () => { })).resolves.toMatchObject({ body: Buffer.from("ok") }); }); + it("allows last-event-id without widening the request header allowlist", async () => { + const port = await startServer((request, response) => { + expect(request.headers["last-event-id"]).toBe("event-1"); + response.end("ok"); + }); + + await expect(requestBoundedHttp(options(port).url, { + ...options(port), + headers: { "Last-Event-ID": "event-1" } + })).resolves.toMatchObject({ body: Buffer.from("ok") }); + }); + it.each([ ["timeoutMs", 0], ["timeoutMs", -1], @@ -268,6 +280,32 @@ describe("requestBoundedHttp", () => { }); }); + it("retains only safe response metadata when a response times out after headers arrive", async () => { + const port = await startServer((_request, response) => { + response.writeHead(200, { + "mcp-session-id": "session-1", + "set-cookie": "secret=value", + "x-internal": "hidden" + }); + response.write("partial"); + }); + + try { + await requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 }); + throw new Error("Expected request to time out."); + } catch (error) { + expect(error).toMatchObject({ + code: "REMOTE_HTTP_TIMEOUT", + message: "Remote HTTP request timed out.", + statusCode: 200, + headers: { "mcp-session-id": "session-1" } + }); + expect((error as { headers?: Record }).headers).toEqual({ + "mcp-session-id": "session-1" + }); + } + }); + it("uses the DNS-pinned local target", async () => { const port = await startServer((_request, response) => response.end("ok")); diff --git a/tests/sse-observation.test.ts b/tests/sse-observation.test.ts new file mode 100644 index 0000000..2aa0b19 --- /dev/null +++ b/tests/sse-observation.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; + +import { observeFirstSseEvent } from "../src/core/sse-observation.js"; + +describe("observeFirstSseEvent", () => { + it("observes the first complete CRLF-delimited event without returning its data", () => { + const observation = observeFirstSseEvent(Buffer.from( + ": ready\r\nid: event-1\r\nretry: 2500\r\ndata: secret payload\r\nunknown: ignored\r\n\r\nid: event-2\r\n\r\n" + )); + + expect(observation).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 2500, + malformed: false + }); + expect(observation).not.toHaveProperty("data"); + }); + + it("does not observe an incomplete event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\ndata: secret payload\n"))).toEqual({ + complete: false, + eventId: null, + retryMs: null, + malformed: false + }); + }); + + it("ignores invalid retry values", () => { + expect(observeFirstSseEvent(Buffer.from("retry: 12ms\nretry: -1\nretry: 125\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: 125, + malformed: false + }); + }); + + it("rejects unsafe event IDs", () => { + expect(observeFirstSseEvent(Buffer.from("id: event\u0001-1\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: true + }); + }); + + it("rejects oversized event ID fields", () => { + expect(observeFirstSseEvent(Buffer.from(`id: ${"a".repeat(1_025)}\n\n`))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: true + }); + }); +}); From fcf3a5f28a19e2f6a0468da45a5fefea262c55d3 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 14:09:02 +0300 Subject: [PATCH 02/15] fix: preserve valid SSE retry observations --- src/core/sse-observation.ts | 5 ++++- tests/bounded-http-client.test.ts | 14 +++++++++---- tests/sse-observation.test.ts | 34 +++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/core/sse-observation.ts b/src/core/sse-observation.ts index a69d756..0408760 100644 --- a/src/core/sse-observation.ts +++ b/src/core/sse-observation.ts @@ -69,7 +69,10 @@ export function observeFirstSseEvent(body: Buffer): SseObservation { malformed = true; } } else if (field.equals(Buffer.from("retry"))) { - retryMs = parseRetry(value); + const parsedRetryMs = parseRetry(value); + if (parsedRetryMs !== null) { + retryMs = parsedRetryMs; + } } } diff --git a/tests/bounded-http-client.test.ts b/tests/bounded-http-client.test.ts index 49e2ec8..63aca90 100644 --- a/tests/bounded-http-client.test.ts +++ b/tests/bounded-http-client.test.ts @@ -274,10 +274,16 @@ describe("requestBoundedHttp", () => { it("times out a non-responsive request", async () => { const port = await startServer(() => undefined); - await expect(requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 })).rejects.toMatchObject({ - code: "REMOTE_HTTP_TIMEOUT", - message: "Remote HTTP request timed out." - }); + try { + await requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 }); + throw new Error("Expected request to time out."); + } catch (error) { + expect(error).toMatchObject({ + code: "REMOTE_HTTP_TIMEOUT", + message: "Remote HTTP request timed out." + }); + expect(error).toMatchObject({ statusCode: undefined, headers: undefined }); + } }); it("retains only safe response metadata when a response times out after headers arrive", async () => { diff --git a/tests/sse-observation.test.ts b/tests/sse-observation.test.ts index 2aa0b19..cb62b70 100644 --- a/tests/sse-observation.test.ts +++ b/tests/sse-observation.test.ts @@ -26,8 +26,17 @@ describe("observeFirstSseEvent", () => { }); }); - it("ignores invalid retry values", () => { - expect(observeFirstSseEvent(Buffer.from("retry: 12ms\nretry: -1\nretry: 125\n\n"))).toEqual({ + it("observes a complete LF-delimited event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\nretry: 125\n\n"))).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 125, + malformed: false + }); + }); + + it("does not replace a valid retry with a later invalid retry", () => { + expect(observeFirstSseEvent(Buffer.from("retry: 125\nretry: nope\n\n"))).toEqual({ complete: true, eventId: null, retryMs: 125, @@ -35,6 +44,27 @@ describe("observeFirstSseEvent", () => { }); }); + it.each([ + ["an overflowing retry", "9007199254740992"], + ["an oversized retry", "9".repeat(1_025)] + ])("ignores %s without replacing a valid retry", (_name, invalidRetry) => { + expect(observeFirstSseEvent(Buffer.from(`retry: 125\nretry: ${invalidRetry}\n\n`))).toEqual({ + complete: true, + eventId: null, + retryMs: 125, + malformed: false + }); + }); + + it("resets the event ID when the event contains an empty id field", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\nid:\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: false + }); + }); + it("rejects unsafe event IDs", () => { expect(observeFirstSseEvent(Buffer.from("id: event\u0001-1\n\n"))).toEqual({ complete: true, From 38b41d8114247e40dbc6952b93faec9b3b4f34d2 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 14:17:48 +0300 Subject: [PATCH 03/15] fix: harden SSE line parsing and timeout tests --- src/core/sse-observation.ts | 8 ++-- tests/bounded-http-client.test.ts | 67 ++++++++++++++++++++++--------- tests/sse-observation.test.ts | 9 +++++ 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/core/sse-observation.ts b/src/core/sse-observation.ts index 0408760..643186d 100644 --- a/src/core/sse-observation.ts +++ b/src/core/sse-observation.ts @@ -35,12 +35,14 @@ export function observeFirstSseEvent(body: Buffer): SseObservation { let lineStart = 0; for (let index = 0; index < body.length; index += 1) { - if (body[index] !== 0x0a) { + if (body[index] !== 0x0a && body[index] !== 0x0d) { continue; } - const lineEnd = index > lineStart && body[index - 1] === 0x0d ? index - 1 : index; - const line = body.subarray(lineStart, lineEnd); + const line = body.subarray(lineStart, index); + if (body[index] === 0x0d && body[index + 1] === 0x0a) { + index += 1; + } lineStart = index + 1; if (line.length === 0) { diff --git a/tests/bounded-http-client.test.ts b/tests/bounded-http-client.test.ts index 63aca90..9dbbbf6 100644 --- a/tests/bounded-http-client.test.ts +++ b/tests/bounded-http-client.test.ts @@ -271,44 +271,73 @@ describe("requestBoundedHttp", () => { await expect(responseClosed).resolves.toBeUndefined(); }); - it("times out a non-responsive request", async () => { - const port = await startServer(() => undefined); + it("times out without response metadata before response headers arrive", async () => { + vi.useFakeTimers(); + const request = new EventEmitter(); + const requestEnd = vi.fn(); + Object.assign(request, { destroy: vi.fn(), end: requestEnd }); + requestMock.mockReturnValueOnce(request as unknown as ClientRequest); try { - await requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 }); - throw new Error("Expected request to time out."); - } catch (error) { - expect(error).toMatchObject({ + const pending = requestBoundedHttp("http://mcp.test/mcp", { + allowLocalNetwork: true, + lookup: localLookup(), + timeoutMs: 20 + }); + await vi.advanceTimersByTimeAsync(0); + expect(requestEnd).toHaveBeenCalledOnce(); + const timedOut = expect(pending).rejects.toMatchObject({ code: "REMOTE_HTTP_TIMEOUT", - message: "Remote HTTP request timed out." + message: "Remote HTTP request timed out.", + statusCode: undefined, + headers: undefined }); - expect(error).toMatchObject({ statusCode: undefined, headers: undefined }); + await vi.advanceTimersByTimeAsync(20); + + await timedOut; + } finally { + vi.useRealTimers(); } }); it("retains only safe response metadata when a response times out after headers arrive", async () => { - const port = await startServer((_request, response) => { - response.writeHead(200, { + vi.useFakeTimers(); + const response = new EventEmitter(); + Object.assign(response, { + destroy: vi.fn(), + headers: { "mcp-session-id": "session-1", "set-cookie": "secret=value", "x-internal": "hidden" - }); - response.write("partial"); + }, + socket: { remoteAddress: "127.0.0.1" }, + statusCode: 200 }); + const request = new EventEmitter(); + Object.assign(request, { + destroy: vi.fn(), + end: () => request.emit("response", response) + }); + requestMock.mockReturnValueOnce(request as unknown as ClientRequest); try { - await requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 }); - throw new Error("Expected request to time out."); - } catch (error) { - expect(error).toMatchObject({ + const pending = requestBoundedHttp("http://mcp.test/mcp", { + allowLocalNetwork: true, + lookup: localLookup(), + timeoutMs: 20 + }); + await vi.advanceTimersByTimeAsync(0); + const timedOut = expect(pending).rejects.toMatchObject({ code: "REMOTE_HTTP_TIMEOUT", message: "Remote HTTP request timed out.", statusCode: 200, headers: { "mcp-session-id": "session-1" } }); - expect((error as { headers?: Record }).headers).toEqual({ - "mcp-session-id": "session-1" - }); + await vi.advanceTimersByTimeAsync(20); + + await timedOut; + } finally { + vi.useRealTimers(); } }); diff --git a/tests/sse-observation.test.ts b/tests/sse-observation.test.ts index cb62b70..de5d034 100644 --- a/tests/sse-observation.test.ts +++ b/tests/sse-observation.test.ts @@ -35,6 +35,15 @@ describe("observeFirstSseEvent", () => { }); }); + it("observes only the first complete CR-delimited event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\rretry: 125\r\rid: event-2\r\r"))).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 125, + malformed: false + }); + }); + it("does not replace a valid retry with a later invalid retry", () => { expect(observeFirstSseEvent(Buffer.from("retry: 125\nretry: nope\n\n"))).toEqual({ complete: true, From cfe0ee9a7a60c7d5cc66bf6c6c1499267eb094b1 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 14:31:11 +0300 Subject: [PATCH 04/15] feat: probe remote MCP transport reliability --- src/core/remote-mcp-probe.ts | 84 ++++- src/core/remote-transport-reliability.ts | 362 +++++++++++++++++++++ tests/mcp-command.test.ts | 12 +- tests/remote-mcp-probe.test.ts | 49 ++- tests/remote-transport-reliability.test.ts | 216 ++++++++++++ 5 files changed, 714 insertions(+), 9 deletions(-) create mode 100644 src/core/remote-transport-reliability.ts create mode 100644 tests/remote-transport-reliability.test.ts diff --git a/src/core/remote-mcp-probe.ts b/src/core/remote-mcp-probe.ts index a19fe22..c1a9703 100644 --- a/src/core/remote-mcp-probe.ts +++ b/src/core/remote-mcp-probe.ts @@ -6,6 +6,11 @@ import { } from "./bounded-http-client.js"; import { RemoteNetworkPolicyError, type RemoteLookup } from "./remote-network-policy.js"; import { checkRemoteOAuthReadiness } from "./remote-oauth-readiness.js"; +import { + createSkippedRemoteTransportReliabilityResult, + probeRemoteTransportReliability, + type RemoteTransportReliabilityResult +} from "./remote-transport-reliability.js"; import { inspectRemoteMcpUrl } from "./remote-url-policy.js"; import type { Finding, RemoteRuntimeScorecard, RuntimeCapabilityStatus } from "../domain/types.js"; import { packageVersion } from "../version.js"; @@ -25,11 +30,13 @@ export interface RemoteMcpProbeOptions { requestTimeoutMs?: number; lookup?: RemoteLookup; request?: RemoteMcpRequest; + allowSessionLifecycle?: boolean; } export interface RemoteMcpProbeResult { findings: Finding[]; scorecard: RemoteRuntimeScorecard; + reliability: RemoteTransportReliabilityResult; } function createScorecard(): RemoteRuntimeScorecard { @@ -54,7 +61,11 @@ function failure( return { id, severity: "fail", message, impact, suggestedFix }; } -function finalize(scorecard: RemoteRuntimeScorecard, findings: Finding[]): RemoteMcpProbeResult { +function finalize( + scorecard: RemoteRuntimeScorecard, + findings: Finding[], + reliability = createSkippedRemoteTransportReliabilityResult() +): RemoteMcpProbeResult { scorecard.overall = findings.some((finding) => finding.severity === "fail") ? "fail" : findings.some((finding) => finding.severity === "warn") @@ -62,7 +73,7 @@ function finalize(scorecard: RemoteRuntimeScorecard, findings: Finding[]): Remot : scorecard.initialize === "pass" || scorecard.authorization === "pass" ? "pass" : "skipped"; - return { findings, scorecard }; + return { findings, scorecard, reliability }; } function isPlainObject(value: unknown): value is JsonObject { @@ -327,5 +338,72 @@ export async function probeRemoteMcpServer( } scorecard.protocolHeaders = "pass"; - return finalize(scorecard, findings); + const reliability = await probeRemoteTransportReliability({ + rawUrl, + protocolVersion: MCP_PROTOCOL_VERSION, + request, + requestTimeoutMs: options.requestTimeoutMs, + requestOptions: { + allowLocalNetwork: options.allowLocalNetwork, + lookup: options.lookup + }, + sessionId, + allowSessionLifecycle: options.allowSessionLifecycle, + reinitialize: async (remainingMs) => { + const restartDeadline = Date.now() + remainingMs; + const restartTimeoutMs = (): number => { + const availableMs = restartDeadline - Date.now(); + if (availableMs <= 0) { + throw new Error("restart deadline elapsed"); + } + return Math.min(availableMs, options.requestTimeoutMs ?? 3_000); + }; + const restartInitialize = await request(rawUrl, { + allowLocalNetwork: options.allowLocalNetwork, + lookup: options.lookup, + timeoutMs: restartTimeoutMs(), + method: "POST", + body: initializeBody, + headers: { + Accept: "application/json, text/event-stream", + "Content-Type": "application/json" + }, + stopAfter: (body) => findSseInitializeResponse(body) !== null + }); + if (restartInitialize.statusCode !== 200) { + throw new Error("restart initialize failed"); + } + const restartContentType = mediaType(responseHeader(restartInitialize.headers, "content-type")); + if (restartContentType !== "application/json" && restartContentType !== "text/event-stream") { + throw new Error("restart initialize content type invalid"); + } + const replacementSessionId = responseHeader(restartInitialize.headers, "mcp-session-id"); + if (replacementSessionId !== null && !validSessionId(replacementSessionId)) { + throw new Error("restart session invalid"); + } + const restartMessage = parseInitializeResponse(restartInitialize.body, restartContentType); + if (!restartMessage || !isValidInitializeResponse(restartMessage)) { + throw new Error("restart initialize result invalid"); + } + const restartInitialized = await request(rawUrl, { + allowLocalNetwork: options.allowLocalNetwork, + lookup: options.lookup, + timeoutMs: restartTimeoutMs(), + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), + headers: { + Accept: "application/json, text/event-stream", + "Content-Type": "application/json", + "MCP-Protocol-Version": MCP_PROTOCOL_VERSION, + ...(replacementSessionId === null ? {} : { "MCP-Session-Id": replacementSessionId }) + } + }); + if (restartInitialized.statusCode < 200 || restartInitialized.statusCode >= 300) { + throw new Error("restart initialized failed"); + } + return replacementSessionId; + } + }); + findings.push(...reliability.findings); + return finalize(scorecard, findings, reliability); } diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts new file mode 100644 index 0000000..a73980a --- /dev/null +++ b/src/core/remote-transport-reliability.ts @@ -0,0 +1,362 @@ +import { + BoundedHttpError, + type BoundedHttpRequestOptions, + type BoundedHttpResponse +} from "./bounded-http-client.js"; +import { observeFirstSseEvent, type SseObservation } from "./sse-observation.js"; +import type { Finding } from "../domain/types.js"; + +const DEFAULT_RELIABILITY_TIMEOUT_MS = 3_000; +const MAX_RELIABILITY_REQUESTS = 4; + +export type RemoteTransportReliabilityStatus = "pass" | "warn" | "fail" | "skipped"; + +export interface RemoteTransportReliabilityScorecard { + getSse: RemoteTransportReliabilityStatus; + sessionPropagation: RemoteTransportReliabilityStatus; + resumability: RemoteTransportReliabilityStatus; + disconnectSafety: RemoteTransportReliabilityStatus; + sessionRestart: RemoteTransportReliabilityStatus; + termination: RemoteTransportReliabilityStatus; + overall: RemoteTransportReliabilityStatus; +} + +export interface RemoteTransportReliabilityResult { + findings: Finding[]; + scorecard: RemoteTransportReliabilityScorecard; +} + +export interface RemoteTransportReliabilityOptions { + rawUrl: string; + protocolVersion: string; + request: (rawUrl: string, options?: BoundedHttpRequestOptions) => Promise; + requestTimeoutMs?: number; + requestOptions?: Pick; + sessionId: string | null; + allowSessionLifecycle?: boolean; + reinitialize: (remainingMs: number) => Promise; +} + +interface RequestResult { + response?: BoundedHttpResponse; + error?: unknown; + restartFailed?: boolean; +} + +function createScorecard(): RemoteTransportReliabilityScorecard { + return { + getSse: "skipped", + sessionPropagation: "skipped", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "skipped" + }; +} + +export function createSkippedRemoteTransportReliabilityResult(): RemoteTransportReliabilityResult { + return { findings: [], scorecard: createScorecard() }; +} + +function finding(id: string, severity: "fail" | "warn", message: string, suggestedFix: string): Finding { + return { + id, + severity, + message, + impact: "Remote MCP transport reliability could not be verified within the configured safety bounds.", + suggestedFix + }; +} + +function responseHeader(headers: BoundedHttpResponse["headers"], name: string): string | null { + const value = headers[name]; + return typeof value === "string" ? value : null; +} + +function mediaType(value: string | null): string | null { + return value?.split(";", 1)[0]?.trim().toLowerCase() ?? null; +} + +function validSessionId(value: string | null): boolean { + return value !== null && /^[\x21-\x7e]+$/.test(value); +} + +function wait(delayMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +function finalize(scorecard: RemoteTransportReliabilityScorecard, findings: Finding[]): RemoteTransportReliabilityResult { + const statuses = Object.entries(scorecard) + .filter(([name]) => name !== "overall") + .map(([, status]) => status); + scorecard.overall = statuses.includes("fail") + ? "fail" + : statuses.includes("warn") + ? "warn" + : statuses.includes("pass") + ? "pass" + : "skipped"; + return { findings, scorecard }; +} + +function isSseTimeout(error: unknown): boolean { + return error instanceof BoundedHttpError + && error.code === "REMOTE_HTTP_TIMEOUT" + && error.statusCode === 200 + && mediaType(responseHeader(error.headers ?? {}, "content-type")) === "text/event-stream"; +} + +function classifySseResponse( + response: BoundedHttpResponse, + failurePrefix: "get" | "resume", + findings: Finding[] +): SseObservation | null { + if (response.statusCode !== 200) { + findings.push(finding( + `plugin.runtime.remote.reliability.${failurePrefix}.status`, + "fail", + "The remote MCP endpoint returned an unsupported SSE transport status.", + "Return HTTP 200 for an accepted SSE transport request." + )); + return null; + } + if (mediaType(responseHeader(response.headers, "content-type")) !== "text/event-stream") { + findings.push(finding( + `plugin.runtime.remote.reliability.${failurePrefix}.content_type`, + "fail", + "The remote MCP endpoint returned a non-SSE media type for an SSE transport request.", + "Return text/event-stream for accepted SSE transport requests." + )); + return null; + } + return observeFirstSseEvent(response.body); +} + +export async function probeRemoteTransportReliability( + options: RemoteTransportReliabilityOptions +): Promise { + const scorecard = createScorecard(); + const findings: Finding[] = []; + const timeoutMs = options.requestTimeoutMs ?? DEFAULT_RELIABILITY_TIMEOUT_MS; + const deadline = Date.now() + Math.min(timeoutMs, DEFAULT_RELIABILITY_TIMEOUT_MS); + let currentSessionId = options.sessionId; + let requestCount = 0; + let restarted = false; + + const send = async ( + method: "GET" | "DELETE", + extraHeaders: Record = {}, + stopAfter?: (body: Buffer) => boolean + ): Promise => { + for (let attempt = 0; attempt < 2; attempt += 1) { + if (requestCount >= MAX_RELIABILITY_REQUESTS) { + return { error: new Error("reliability request ceiling reached") }; + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + return { error: new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote reliability deadline elapsed.") }; + } + const sessionId = currentSessionId; + const headers = { + Accept: "text/event-stream", + "MCP-Protocol-Version": options.protocolVersion, + ...(sessionId === null ? {} : { "MCP-Session-Id": sessionId }), + ...extraHeaders + }; + if (sessionId !== null) { + scorecard.sessionPropagation = "pass"; + } + requestCount += 1; + let response: BoundedHttpResponse; + try { + response = await options.request(options.rawUrl, { + ...options.requestOptions, + timeoutMs: Math.max(1, Math.min(remainingMs, DEFAULT_RELIABILITY_TIMEOUT_MS)), + method, + headers, + stopAfter + }); + } catch (error) { + return { error }; + } + + if (sessionId !== null && response.statusCode === 404) { + if (restarted) { + scorecard.sessionRestart = "fail"; + return { restartFailed: true }; + } + restarted = true; + currentSessionId = null; + try { + const replacementSessionId = await options.reinitialize(Math.max(1, deadline - Date.now())); + if (replacementSessionId !== null && !validSessionId(replacementSessionId)) { + throw new Error("invalid replacement session"); + } + currentSessionId = replacementSessionId; + scorecard.sessionRestart = "pass"; + } catch { + scorecard.sessionRestart = "fail"; + return { restartFailed: true }; + } + continue; + } + return { response }; + } + return { restartFailed: true }; + }; + + const first = await send("GET", {}, (body) => observeFirstSseEvent(body).complete); + if (first.restartFailed) { + findings.push(finding( + "plugin.runtime.remote.reliability.session_restart.failed", + "fail", + "The remote MCP session expired and could not be restarted within the bounded probe.", + "Accept a fresh initialize sequence after an expired MCP session." + )); + return finalize(scorecard, findings); + } + if (first.error) { + if (isSseTimeout(first.error)) { + scorecard.getSse = "warn"; + scorecard.disconnectSafety = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.inconclusive", + "warn", + "The remote MCP SSE stream did not produce a complete event before the bounded observation deadline.", + "Emit complete SSE event frames promptly when server-to-client streaming is supported." + )); + return finalize(scorecard, findings); + } + scorecard.getSse = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.failed", + "fail", + "The remote MCP SSE transport request could not be completed.", + "Keep the Streamable HTTP transport reachable within the configured request bounds." + )); + return finalize(scorecard, findings); + } + const firstResponse = first.response; + if (!firstResponse) { + scorecard.getSse = "fail"; + return finalize(scorecard, findings); + } + if (firstResponse.statusCode === 405) { + scorecard.getSse = "pass"; + } else { + const observation = classifySseResponse(firstResponse, "get", findings); + if (observation === null) { + scorecard.getSse = "fail"; + return finalize(scorecard, findings); + } + if (!observation.complete) { + scorecard.getSse = "warn"; + scorecard.disconnectSafety = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.inconclusive", + "warn", + "The remote MCP SSE response ended before a complete event could be observed.", + "Emit complete SSE event frames when server-to-client streaming is supported." + )); + return finalize(scorecard, findings); + } + if (observation.malformed) { + scorecard.getSse = "fail"; + scorecard.disconnectSafety = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.malformed", + "fail", + "The remote MCP endpoint emitted malformed SSE event framing.", + "Return complete SSE events with valid id and retry fields." + )); + return finalize(scorecard, findings); + } + scorecard.getSse = "pass"; + scorecard.disconnectSafety = "pass"; + + if (observation.eventId !== null) { + const remainingMs = deadline - Date.now(); + if (observation.retryMs !== null) { + if (observation.retryMs >= remainingMs) { + scorecard.resumability = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.resume.inconclusive", + "warn", + "The remote MCP SSE retry delay exceeded the remaining bounded observation deadline.", + "Advertise an SSE retry delay that fits within the configured probe deadline." + )); + return finalize(scorecard, findings); + } + await wait(observation.retryMs); + } + const resumed = await send("GET", { "Last-Event-ID": observation.eventId }, (body) => observeFirstSseEvent(body).complete); + if (resumed.restartFailed) { + findings.push(finding( + "plugin.runtime.remote.reliability.session_restart.failed", + "fail", + "The remote MCP session expired during SSE resumability validation and could not be restarted.", + "Accept one fresh initialize sequence after an expired MCP session." + )); + return finalize(scorecard, findings); + } + if (resumed.error) { + scorecard.resumability = isSseTimeout(resumed.error) ? "warn" : "fail"; + findings.push(finding( + isSseTimeout(resumed.error) + ? "plugin.runtime.remote.reliability.resume.inconclusive" + : "plugin.runtime.remote.reliability.resume.failed", + isSseTimeout(resumed.error) ? "warn" : "fail", + "The remote MCP SSE resume request could not be completed within the bounded probe.", + "Accept one bounded SSE reconnect using Last-Event-ID." + )); + return finalize(scorecard, findings); + } + if (!resumed.response) { + scorecard.resumability = "fail"; + return finalize(scorecard, findings); + } + const resumeObservation = classifySseResponse(resumed.response, "resume", findings); + if (resumeObservation === null || resumeObservation.malformed) { + scorecard.resumability = "fail"; + if (resumeObservation?.malformed) { + findings.push(finding( + "plugin.runtime.remote.reliability.resume.malformed", + "fail", + "The remote MCP endpoint emitted malformed SSE framing after a resume request.", + "Return complete SSE events with valid id and retry fields after reconnecting." + )); + } + return finalize(scorecard, findings); + } + scorecard.resumability = resumeObservation.complete ? "pass" : "warn"; + if (!resumeObservation.complete) { + findings.push(finding( + "plugin.runtime.remote.reliability.resume.inconclusive", + "warn", + "The remote MCP SSE resume response ended before a complete event could be observed.", + "Emit complete SSE event frames after accepting a reconnect." + )); + return finalize(scorecard, findings); + } + } + } + + if (!options.allowSessionLifecycle || currentSessionId === null) { + return finalize(scorecard, findings); + } + const terminated = await send("DELETE"); + if (terminated.restartFailed || terminated.error || !terminated.response + || (terminated.response.statusCode !== 405 && (terminated.response.statusCode < 200 || terminated.response.statusCode >= 300))) { + scorecard.termination = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.termination.failed", + "fail", + "The remote MCP session could not be terminated with the approved lifecycle request.", + "Return a successful response or HTTP 405 for a bounded MCP session DELETE request." + )); + return finalize(scorecard, findings); + } + scorecard.termination = "pass"; + return finalize(scorecard, findings); +} diff --git a/tests/mcp-command.test.ts b/tests/mcp-command.test.ts index dc6bc24..f752707 100644 --- a/tests/mcp-command.test.ts +++ b/tests/mcp-command.test.ts @@ -46,6 +46,12 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { const chunks: Buffer[] = []; request.on("data", (chunk: Buffer) => chunks.push(chunk)); request.on("end", () => { + if (request.method === "GET") { + requests.push("GET"); + response.writeHead(405); + response.end(); + return; + } const message = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { method: string }; requests.push(message.method); @@ -118,7 +124,7 @@ describe("mcp command", () => { } } }); - expect(remote.requests).toEqual(["initialize", "notifications/initialized"]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await remote.close(); } @@ -141,7 +147,7 @@ describe("mcp command", () => { expect(JSON.parse(stdout.join(""))).toMatchObject({ runtimeScorecard: { remote: { networkSafety: "pass", overall: "pass" } } }); - expect(remote.requests).toEqual(["initialize", "notifications/initialized"]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await remote.close(); } @@ -171,7 +177,7 @@ describe("mcp command", () => { overall: "fail" }); expect(failing.requests).toEqual(["initialize"]); - expect(passing.requests).toEqual(["initialize", "notifications/initialized"]); + expect(passing.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await failing.close(); await passing.close(); diff --git a/tests/remote-mcp-probe.test.ts b/tests/remote-mcp-probe.test.ts index 651a5d3..01eb00f 100644 --- a/tests/remote-mcp-probe.test.ts +++ b/tests/remote-mcp-probe.test.ts @@ -71,6 +71,11 @@ describe("probeRemoteMcpServer", () => { request.on("data", (chunk) => { body += chunk; }); request.on("end", () => { requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } const message = JSON.parse(body) as { id?: number; method: string }; if (message.method === "initialize") { response.writeHead(200, { @@ -98,9 +103,9 @@ describe("probeRemoteMcpServer", () => { authorization: "skipped", overall: "pass" }); - expect(requests).toHaveLength(2); - expect(requests.map((request) => request.method)).toEqual(["POST", "POST"]); - expect(requests.map((request) => JSON.parse(request.body).method)).toEqual([ + expect(requests).toHaveLength(3); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET"]); + expect(requests.slice(0, 2).map((request) => JSON.parse(request.body).method)).toEqual([ "initialize", "notifications/initialized" ]); @@ -120,15 +125,53 @@ describe("probeRemoteMcpServer", () => { expect(requests[0]?.headers.cookie).toBeUndefined(); expect(requests[1]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); expect(requests[1]?.headers["mcp-session-id"]).toBe("session-secret-sentinel"); + expect(requests[2]?.headers["mcp-session-id"]).toBe("session-secret-sentinel"); assertPrivate(result); }); + it("treats a bounded SSE GET 405 as protocol-compliant after initialization", async () => { + const requests: Array<{ method: string; headers: Record; body: string }> = []; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([]); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET"]); + expect(requests[2]?.headers.accept).toBe("text/event-stream"); + expect(requests[2]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); + }); + it("skips SSE primer events until the initialize response without waiting for the stream to close", async () => { const port = await startServer((request, response) => { let body = ""; request.setEncoding("utf8"); request.on("data", (chunk) => { body += chunk; }); request.on("end", () => { + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } const message = JSON.parse(body) as { id?: number; method: string }; if (message.method === "initialize") { response.writeHead(200, { "content-type": "text/event-stream" }); diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts new file mode 100644 index 0000000..56728e7 --- /dev/null +++ b/tests/remote-transport-reliability.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it } from "vitest"; + +import { BoundedHttpError, type BoundedHttpRequestOptions, type BoundedHttpResponse } from "../src/core/bounded-http-client.js"; +import { probeRemoteTransportReliability } from "../src/core/remote-transport-reliability.js"; + +const endpoint = "https://mcp.example/mcp"; +const protocolVersion = "2025-11-25"; + +function response(statusCode: number, contentType?: string, body = ""): BoundedHttpResponse { + return { + statusCode, + headers: contentType === undefined ? {} : { "content-type": contentType }, + body: Buffer.from(body) + }; +} + +function scriptedRequest(responses: Array) { + const requests: BoundedHttpRequestOptions[] = []; + return { + requests, + request: async (_url: string, options?: BoundedHttpRequestOptions): Promise => { + requests.push(options ?? {}); + const next = responses.shift(); + if (next instanceof Error) throw next; + if (next === undefined) throw new Error("unexpected request"); + return next; + } + }; +} + +function probe(request: (url: string, options?: BoundedHttpRequestOptions) => Promise, overrides: { + sessionId?: string | null; + allowSessionLifecycle?: boolean; + requestTimeoutMs?: number; + reinitialize?: () => Promise; +} = {}) { + return probeRemoteTransportReliability({ + rawUrl: endpoint, + protocolVersion, + request, + requestTimeoutMs: overrides.requestTimeoutMs ?? 50, + sessionId: overrides.sessionId ?? null, + allowSessionLifecycle: overrides.allowSessionLifecycle, + reinitialize: overrides.reinitialize ?? (async () => null) + }); +} + +function assertRedacted(value: unknown): void { + const serialized = JSON.stringify(value); + expect(serialized).not.toContain("session-secret-sentinel"); + expect(serialized).not.toContain("event-secret-sentinel"); + expect(serialized).not.toContain("retry-secret-sentinel"); + expect(serialized).not.toContain("remote-body-secret-sentinel"); +} + +describe("probeRemoteTransportReliability", () => { + it("accepts GET 405 as compliant without attempting resume", async () => { + const fixture = scriptedRequest([response(405)]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", resumability: "skipped", overall: "pass" }); + expect(fixture.requests).toHaveLength(1); + }); + + it("uses a safe event id for exactly one resume without retaining it", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\ndata: remote-body-secret-sentinel\n\n"), + response(200, "text/event-stream", "\n") + ]); + + const result = await probe(fixture.request, { sessionId: "session-secret-sentinel" }); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", sessionPropagation: "pass", resumability: "pass", overall: "pass" }); + expect(fixture.requests).toHaveLength(2); + expect(fixture.requests[0]?.headers).toMatchObject({ + Accept: "text/event-stream", + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel" + }); + expect(fixture.requests[1]?.headers).toMatchObject({ "Last-Event-ID": "event-secret-sentinel" }); + assertRedacted(result); + }); + + it("accepts a complete SSE event without an id without attempting resume", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "event: message\ndata: remote-body-secret-sentinel\n\n") + ]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", resumability: "skipped", overall: "pass" }); + expect(fixture.requests).toHaveLength(1); + assertRedacted(result); + }); + + it("honors a retry delay only when it remains within the reliability deadline", async () => { + const withinDeadline = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\nretry: 1\n\n"), + response(200, "text/event-stream", "\n") + ]); + const withinResult = await probe(withinDeadline.request, { requestTimeoutMs: 50 }); + + const overDeadline = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\nretry: 999999\n\n") + ]); + const overResult = await probe(overDeadline.request, { requestTimeoutMs: 10 }); + + expect(withinResult.scorecard.resumability).toBe("pass"); + expect(withinDeadline.requests).toHaveLength(2); + expect(overResult.scorecard.resumability).toBe("warn"); + expect(overDeadline.requests).toHaveLength(1); + assertRedacted(overResult); + }); + + it.each([ + [response(500), "plugin.runtime.remote.reliability.get.status"], + [response(200, "application/json"), "plugin.runtime.remote.reliability.get.content_type"], + [response(200, "text/event-stream", "id: invalid event-secret-sentinel\n\n"), "plugin.runtime.remote.reliability.get.malformed"] + ])("fails invalid GET evidence with stable redacted findings", async (fixtureResponse, findingId) => { + const fixture = scriptedRequest([fixtureResponse]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([expect.objectContaining({ id: findingId, severity: "fail" })]); + expect(result.scorecard.overall).toBe("fail"); + assertRedacted(result); + }); + + it("treats an incomplete SSE response after headers as inconclusive", async () => { + const timeout = new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "timeout", 200, { "content-type": "text/event-stream" }); + const fixture = scriptedRequest([timeout]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.get.inconclusive", severity: "warn" })]); + expect(result.scorecard.overall).toBe("warn"); + }); + + it("reinitializes once after a session-bound GET 404 and uses the replacement session", async () => { + const fixture = scriptedRequest([response(404), response(405)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(2); + expect(fixture.requests[1]?.headers?.["MCP-Session-Id"]).toBe("replacement-session-secret-sentinel"); + expect(result.scorecard).toMatchObject({ sessionRestart: "pass", overall: "pass" }); + assertRedacted(result); + }); + + it("fails a single bounded session restart without recursion", async () => { + const fixture = scriptedRequest([response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + throw new Error("session-secret-sentinel"); + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(1); + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" })]); + assertRedacted(result); + }); + + it("does not restart a replacement session or exceed the request ceiling", async () => { + const fixture = scriptedRequest([response(404), response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(2); + expect(result.scorecard.sessionRestart).toBe("fail"); + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" })]); + assertRedacted(result); + }); + + it.each([ + [false, response(405), "skipped", "pass"], + [true, response(204), "pass", "pass"], + [true, response(405), "pass", "pass"], + [true, response(500), "fail", "fail"] + ])("limits DELETE lifecycle behavior to explicit consent", async (allowSessionLifecycle, deleteResponse, termination, overall) => { + const fixture = scriptedRequest([response(405), deleteResponse]); + + const result = await probe(fixture.request, { sessionId: "session-secret-sentinel", allowSessionLifecycle }); + + expect(result.scorecard.termination).toBe(termination); + expect(result.scorecard.overall).toBe(overall); + expect(fixture.requests.map((request) => request.method)).toEqual( + allowSessionLifecycle ? ["GET", "DELETE"] : ["GET"] + ); + assertRedacted(result); + }); +}); From c081b0a600435dd3f9ce00feb6c8c630f15b2bad Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 14:45:25 +0300 Subject: [PATCH 05/15] fix: bound remote reliability restart flow --- src/core/remote-mcp-probe.ts | 21 +---- src/core/remote-transport-reliability.ts | 38 +++++---- tests/remote-mcp-probe.test.ts | 98 ++++++++++++++++++++++ tests/remote-transport-reliability.test.ts | 92 ++++++++++++++++++-- 4 files changed, 209 insertions(+), 40 deletions(-) diff --git a/src/core/remote-mcp-probe.ts b/src/core/remote-mcp-probe.ts index c1a9703..273009e 100644 --- a/src/core/remote-mcp-probe.ts +++ b/src/core/remote-mcp-probe.ts @@ -349,19 +349,8 @@ export async function probeRemoteMcpServer( }, sessionId, allowSessionLifecycle: options.allowSessionLifecycle, - reinitialize: async (remainingMs) => { - const restartDeadline = Date.now() + remainingMs; - const restartTimeoutMs = (): number => { - const availableMs = restartDeadline - Date.now(); - if (availableMs <= 0) { - throw new Error("restart deadline elapsed"); - } - return Math.min(availableMs, options.requestTimeoutMs ?? 3_000); - }; - const restartInitialize = await request(rawUrl, { - allowLocalNetwork: options.allowLocalNetwork, - lookup: options.lookup, - timeoutMs: restartTimeoutMs(), + reinitialize: async (requestWithinBudget) => { + const restartInitialize = await requestWithinBudget({ method: "POST", body: initializeBody, headers: { @@ -385,10 +374,7 @@ export async function probeRemoteMcpServer( if (!restartMessage || !isValidInitializeResponse(restartMessage)) { throw new Error("restart initialize result invalid"); } - const restartInitialized = await request(rawUrl, { - allowLocalNetwork: options.allowLocalNetwork, - lookup: options.lookup, - timeoutMs: restartTimeoutMs(), + const restartInitialized = await requestWithinBudget({ method: "POST", body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), headers: { @@ -404,6 +390,5 @@ export async function probeRemoteMcpServer( return replacementSessionId; } }); - findings.push(...reliability.findings); return finalize(scorecard, findings, reliability); } diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts index a73980a..2dd3e58 100644 --- a/src/core/remote-transport-reliability.ts +++ b/src/core/remote-transport-reliability.ts @@ -7,7 +7,7 @@ import { observeFirstSseEvent, type SseObservation } from "./sse-observation.js" import type { Finding } from "../domain/types.js"; const DEFAULT_RELIABILITY_TIMEOUT_MS = 3_000; -const MAX_RELIABILITY_REQUESTS = 4; +const MAX_RELIABILITY_REQUESTS = 6; export type RemoteTransportReliabilityStatus = "pass" | "warn" | "fail" | "skipped"; @@ -26,6 +26,10 @@ export interface RemoteTransportReliabilityResult { scorecard: RemoteTransportReliabilityScorecard; } +export type RemoteTransportReliabilityRequest = ( + options: BoundedHttpRequestOptions +) => Promise; + export interface RemoteTransportReliabilityOptions { rawUrl: string; protocolVersion: string; @@ -34,7 +38,7 @@ export interface RemoteTransportReliabilityOptions { requestOptions?: Pick; sessionId: string | null; allowSessionLifecycle?: boolean; - reinitialize: (remainingMs: number) => Promise; + reinitialize: (request: RemoteTransportReliabilityRequest) => Promise; } interface RequestResult { @@ -144,19 +148,28 @@ export async function probeRemoteTransportReliability( let requestCount = 0; let restarted = false; + const requestWithinBudget: RemoteTransportReliabilityRequest = async (requestOptions) => { + if (requestCount >= MAX_RELIABILITY_REQUESTS) { + throw new Error("reliability request ceiling reached"); + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote reliability deadline elapsed."); + } + requestCount += 1; + return options.request(options.rawUrl, { + ...requestOptions, + ...options.requestOptions, + timeoutMs: Math.max(1, Math.min(remainingMs, DEFAULT_RELIABILITY_TIMEOUT_MS)) + }); + }; + const send = async ( method: "GET" | "DELETE", extraHeaders: Record = {}, stopAfter?: (body: Buffer) => boolean ): Promise => { for (let attempt = 0; attempt < 2; attempt += 1) { - if (requestCount >= MAX_RELIABILITY_REQUESTS) { - return { error: new Error("reliability request ceiling reached") }; - } - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { - return { error: new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote reliability deadline elapsed.") }; - } const sessionId = currentSessionId; const headers = { Accept: "text/event-stream", @@ -167,12 +180,9 @@ export async function probeRemoteTransportReliability( if (sessionId !== null) { scorecard.sessionPropagation = "pass"; } - requestCount += 1; let response: BoundedHttpResponse; try { - response = await options.request(options.rawUrl, { - ...options.requestOptions, - timeoutMs: Math.max(1, Math.min(remainingMs, DEFAULT_RELIABILITY_TIMEOUT_MS)), + response = await requestWithinBudget({ method, headers, stopAfter @@ -189,7 +199,7 @@ export async function probeRemoteTransportReliability( restarted = true; currentSessionId = null; try { - const replacementSessionId = await options.reinitialize(Math.max(1, deadline - Date.now())); + const replacementSessionId = await options.reinitialize(requestWithinBudget); if (replacementSessionId !== null && !validSessionId(replacementSessionId)) { throw new Error("invalid replacement session"); } diff --git a/tests/remote-mcp-probe.test.ts b/tests/remote-mcp-probe.test.ts index 01eb00f..122f290 100644 --- a/tests/remote-mcp-probe.test.ts +++ b/tests/remote-mcp-probe.test.ts @@ -161,6 +161,104 @@ describe("probeRemoteMcpServer", () => { expect(requests[2]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); }); + it("keeps reliability failures internal until the public scorecard is wired", async () => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(500, { "content-type": "application/json" }); + response.end('{"error":"remote-body-secret-sentinel"}'); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([]); + expect(result.scorecard.overall).toBe("pass"); + expect(result.reliability.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.get.status", severity: "fail" }) + ]); + expect(result.reliability.scorecard.overall).toBe("fail"); + assertPrivate(result); + }); + + it("shares one fixed request budget with both actual session-restart POSTs", async () => { + const requests: Array<{ method: string; headers: Record; body: string }> = []; + let initializeCount = 0; + let getCount = 0; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + getCount += 1; + if (getCount === 1) { + response.writeHead(404); + } else if (getCount === 2) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("id: event-secret-sentinel\n\n"); + return; + } else if (getCount === 3) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("\n"); + return; + } else { + response.writeHead(500); + } + response.end(); + return; + } + if (request.method === "DELETE") { + response.writeHead(204); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + initializeCount += 1; + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": initializeCount === 1 + ? "session-secret-sentinel" + : "replacement-session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, { + ...options(port), + allowSessionLifecycle: true + }); + + expect(result.findings).toEqual([]); + expect(result.reliability.scorecard).toMatchObject({ sessionRestart: "pass", resumability: "pass", termination: "pass", overall: "pass" }); + expect(requests.map((request) => request.method)).toEqual([ + "POST", "POST", "GET", "POST", "POST", "GET", "GET", "DELETE" + ]); + expect(requests).toHaveLength(8); + expect(initializeCount).toBe(2); + assertPrivate(result); + }); + it("skips SSE primer events until the initialize response without waiting for the stream to close", async () => { const port = await startServer((request, response) => { let body = ""; diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index 56728e7..b281408 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -47,10 +47,14 @@ function probe(request: (url: string, options?: BoundedHttpRequestOptions) => Pr function assertRedacted(value: unknown): void { const serialized = JSON.stringify(value); - expect(serialized).not.toContain("session-secret-sentinel"); - expect(serialized).not.toContain("event-secret-sentinel"); - expect(serialized).not.toContain("retry-secret-sentinel"); - expect(serialized).not.toContain("remote-body-secret-sentinel"); + for (const secret of [ + "session-secret-sentinel", + "event-secret-sentinel", + "999999", + "remote-body-secret-sentinel" + ]) { + expect(serialized).not.toContain(secret); + } } describe("probeRemoteTransportReliability", () => { @@ -67,20 +71,31 @@ describe("probeRemoteTransportReliability", () => { it("uses a safe event id for exactly one resume without retaining it", async () => { const fixture = scriptedRequest([ response(200, "text/event-stream", "id: event-secret-sentinel\ndata: remote-body-secret-sentinel\n\n"), - response(200, "text/event-stream", "\n") + response(200, "text/event-stream", "\n"), + response(204) ]); - const result = await probe(fixture.request, { sessionId: "session-secret-sentinel" }); + const result = await probe(fixture.request, { sessionId: "session-secret-sentinel", allowSessionLifecycle: true }); expect(result.findings).toEqual([]); expect(result.scorecard).toMatchObject({ getSse: "pass", sessionPropagation: "pass", resumability: "pass", overall: "pass" }); - expect(fixture.requests).toHaveLength(2); + expect(fixture.requests).toHaveLength(3); expect(fixture.requests[0]?.headers).toMatchObject({ Accept: "text/event-stream", "MCP-Protocol-Version": protocolVersion, "MCP-Session-Id": "session-secret-sentinel" }); - expect(fixture.requests[1]?.headers).toMatchObject({ "Last-Event-ID": "event-secret-sentinel" }); + expect(fixture.requests[0]?.headers?.["Last-Event-ID"]).toBeUndefined(); + expect(fixture.requests[1]?.headers).toMatchObject({ + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel", + "Last-Event-ID": "event-secret-sentinel" + }); + expect(fixture.requests[2]?.headers).toMatchObject({ + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel" + }); + expect(fixture.requests[2]?.headers?.["Last-Event-ID"]).toBeUndefined(); assertRedacted(result); }); @@ -159,6 +174,67 @@ describe("probeRemoteTransportReliability", () => { assertRedacted(result); }); + it("counts both restart POST requests against the shared reliability request budget", async () => { + const fixture = scriptedRequest([ + response(404), + response(200, "application/json", "{}"), + response(202), + response(405) + ]); + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async (requestWithinBudget: unknown) => { + if (typeof requestWithinBudget !== "function") { + throw new Error("restart requests must share the reliability budget"); + } + const bounded = requestWithinBudget as (options: BoundedHttpRequestOptions) => Promise; + await bounded({ method: "POST", headers: { "Content-Type": "application/json" } }); + await bounded({ method: "POST", headers: { "Content-Type": "application/json" } }); + return "replacement-session-secret-sentinel"; + } + }); + + expect(result.scorecard).toMatchObject({ sessionRestart: "pass", overall: "pass" }); + expect(fixture.requests.map((request) => request.method)).toEqual(["GET", "POST", "POST", "GET"]); + expect(fixture.requests).toHaveLength(4); + assertRedacted(result); + }); + + it("refuses a seventh request through the restart budget", async () => { + const fixture = scriptedRequest([ + response(404), + response(202), + response(202), + response(202), + response(202), + response(202) + ]); + let ceilingReached = false; + + await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async (requestWithinBudget: unknown) => { + if (typeof requestWithinBudget !== "function") { + throw new Error("restart requests must share the reliability budget"); + } + const bounded = requestWithinBudget as (options: BoundedHttpRequestOptions) => Promise; + for (let index = 0; index < 5; index += 1) { + await bounded({ method: "POST" }); + } + try { + await bounded({ method: "POST" }); + } catch { + ceilingReached = true; + } + return "replacement-session-secret-sentinel"; + } + }); + + expect(ceilingReached).toBe(true); + expect(fixture.requests).toHaveLength(6); + }); + it("fails a single bounded session restart without recursion", async () => { const fixture = scriptedRequest([response(404)]); let restarts = 0; From 6d5f1da674acd03578cae47341ec125d1a90896e Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 15:01:58 +0300 Subject: [PATCH 06/15] feat: report remote transport reliability --- src/core/output-contract.ts | 27 ++++++ src/core/remote-mcp-probe.ts | 14 ++- src/core/remote-transport-reliability.ts | 18 +--- src/core/runtime-probe.ts | 20 ++++- src/domain/types.ts | 11 +++ src/reporting/render-markdown-report.ts | 14 +++ src/reporting/render-text-report.ts | 12 +++ src/rules/rule-catalog.ts | 108 +++++++++++++++++++++++ tests/json-runtime-scorecard.test.ts | 59 +++++++++++++ tests/markdown-report.test.ts | 21 +++++ tests/remote-mcp-probe.test.ts | 89 +++++++++++++++++-- tests/render-text-report.test.ts | 21 +++++ tests/rule-catalog.test.ts | 21 +++++ 13 files changed, 400 insertions(+), 35 deletions(-) diff --git a/src/core/output-contract.ts b/src/core/output-contract.ts index c3e7d7f..82dc1b5 100644 --- a/src/core/output-contract.ts +++ b/src/core/output-contract.ts @@ -81,6 +81,32 @@ const runtimeConformanceSchema = { additionalProperties: false }; +const remoteTransportReliabilitySchema = { + type: "object", + properties: { + getSse: runtimeCapabilityStatusSchema, + sessionPropagation: runtimeCapabilityStatusSchema, + resumability: runtimeCapabilityStatusSchema, + disconnectSafety: runtimeCapabilityStatusSchema, + sessionRestart: runtimeCapabilityStatusSchema, + termination: runtimeCapabilityStatusSchema, + overall: { + type: "string", + enum: ["pass", "warn", "fail", "skipped"] + } + }, + required: [ + "getSse", + "sessionPropagation", + "resumability", + "disconnectSafety", + "sessionRestart", + "termination", + "overall" + ], + additionalProperties: false +}; + const remoteRuntimeScorecardSchema = { type: "object", properties: { @@ -94,6 +120,7 @@ const remoteRuntimeScorecardSchema = { }, protocolHeaders: runtimeCapabilityStatusSchema, authorization: runtimeCapabilityStatusSchema, + reliability: remoteTransportReliabilitySchema, overall: { type: "string", enum: ["pass", "warn", "fail", "skipped"] diff --git a/src/core/remote-mcp-probe.ts b/src/core/remote-mcp-probe.ts index 273009e..af39972 100644 --- a/src/core/remote-mcp-probe.ts +++ b/src/core/remote-mcp-probe.ts @@ -7,9 +7,7 @@ import { import { RemoteNetworkPolicyError, type RemoteLookup } from "./remote-network-policy.js"; import { checkRemoteOAuthReadiness } from "./remote-oauth-readiness.js"; import { - createSkippedRemoteTransportReliabilityResult, - probeRemoteTransportReliability, - type RemoteTransportReliabilityResult + probeRemoteTransportReliability } from "./remote-transport-reliability.js"; import { inspectRemoteMcpUrl } from "./remote-url-policy.js"; import type { Finding, RemoteRuntimeScorecard, RuntimeCapabilityStatus } from "../domain/types.js"; @@ -36,7 +34,6 @@ export interface RemoteMcpProbeOptions { export interface RemoteMcpProbeResult { findings: Finding[]; scorecard: RemoteRuntimeScorecard; - reliability: RemoteTransportReliabilityResult; } function createScorecard(): RemoteRuntimeScorecard { @@ -63,8 +60,7 @@ function failure( function finalize( scorecard: RemoteRuntimeScorecard, - findings: Finding[], - reliability = createSkippedRemoteTransportReliabilityResult() + findings: Finding[] ): RemoteMcpProbeResult { scorecard.overall = findings.some((finding) => finding.severity === "fail") ? "fail" @@ -73,7 +69,7 @@ function finalize( : scorecard.initialize === "pass" || scorecard.authorization === "pass" ? "pass" : "skipped"; - return { findings, scorecard, reliability }; + return { findings, scorecard }; } function isPlainObject(value: unknown): value is JsonObject { @@ -390,5 +386,7 @@ export async function probeRemoteMcpServer( return replacementSessionId; } }); - return finalize(scorecard, findings, reliability); + scorecard.reliability = reliability.scorecard; + findings.push(...reliability.findings); + return finalize(scorecard, findings); } diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts index 2dd3e58..db2fb53 100644 --- a/src/core/remote-transport-reliability.ts +++ b/src/core/remote-transport-reliability.ts @@ -4,23 +4,11 @@ import { type BoundedHttpResponse } from "./bounded-http-client.js"; import { observeFirstSseEvent, type SseObservation } from "./sse-observation.js"; -import type { Finding } from "../domain/types.js"; +import type { Finding, RemoteTransportReliabilityScorecard } from "../domain/types.js"; const DEFAULT_RELIABILITY_TIMEOUT_MS = 3_000; const MAX_RELIABILITY_REQUESTS = 6; -export type RemoteTransportReliabilityStatus = "pass" | "warn" | "fail" | "skipped"; - -export interface RemoteTransportReliabilityScorecard { - getSse: RemoteTransportReliabilityStatus; - sessionPropagation: RemoteTransportReliabilityStatus; - resumability: RemoteTransportReliabilityStatus; - disconnectSafety: RemoteTransportReliabilityStatus; - sessionRestart: RemoteTransportReliabilityStatus; - termination: RemoteTransportReliabilityStatus; - overall: RemoteTransportReliabilityStatus; -} - export interface RemoteTransportReliabilityResult { findings: Finding[]; scorecard: RemoteTransportReliabilityScorecard; @@ -59,10 +47,6 @@ function createScorecard(): RemoteTransportReliabilityScorecard { }; } -export function createSkippedRemoteTransportReliabilityResult(): RemoteTransportReliabilityResult { - return { findings: [], scorecard: createScorecard() }; -} - function finding(id: string, severity: "fail" | "warn", message: string, suggestedFix: string): Finding { return { id, diff --git a/src/core/runtime-probe.ts b/src/core/runtime-probe.ts index b851312..69a516f 100644 --- a/src/core/runtime-probe.ts +++ b/src/core/runtime-probe.ts @@ -539,6 +539,23 @@ function mergeRemoteRuntimeScorecards( "present-valid": 1, absent: 0 }; + const reliability = left.reliability && right.reliability + ? { + getSse: worstRuntimeStatus(left.reliability.getSse, right.reliability.getSse), + sessionPropagation: worstRuntimeStatus( + left.reliability.sessionPropagation, + right.reliability.sessionPropagation + ), + resumability: worstRuntimeStatus(left.reliability.resumability, right.reliability.resumability), + disconnectSafety: worstRuntimeStatus( + left.reliability.disconnectSafety, + right.reliability.disconnectSafety + ), + sessionRestart: worstRuntimeStatus(left.reliability.sessionRestart, right.reliability.sessionRestart), + termination: worstRuntimeStatus(left.reliability.termination, right.reliability.termination), + overall: worstRuntimeStatus(left.reliability.overall, right.reliability.overall) + } + : left.reliability ?? right.reliability; return { transport: worstRuntimeStatus(left.transport, right.transport), @@ -550,7 +567,8 @@ function mergeRemoteRuntimeScorecards( : right.session, protocolHeaders: worstRuntimeStatus(left.protocolHeaders, right.protocolHeaders), authorization: worstRuntimeStatus(left.authorization, right.authorization), - overall: worstRuntimeStatus(left.overall, right.overall) + overall: worstRuntimeStatus(left.overall, right.overall), + ...(reliability ? { reliability } : {}) }; } diff --git a/src/domain/types.ts b/src/domain/types.ts index e7ab7d4..c8be2ed 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -124,6 +124,17 @@ export interface RemoteRuntimeScorecard { session: "absent" | "present-valid" | "present-invalid"; protocolHeaders: RuntimeCapabilityStatus; authorization: RuntimeCapabilityStatus; + reliability?: RemoteTransportReliabilityScorecard; + overall: "pass" | "warn" | "fail" | "skipped"; +} + +export interface RemoteTransportReliabilityScorecard { + getSse: RuntimeCapabilityStatus; + sessionPropagation: RuntimeCapabilityStatus; + resumability: RuntimeCapabilityStatus; + disconnectSafety: RuntimeCapabilityStatus; + sessionRestart: RuntimeCapabilityStatus; + termination: RuntimeCapabilityStatus; overall: "pass" | "warn" | "fail" | "skipped"; } diff --git a/src/reporting/render-markdown-report.ts b/src/reporting/render-markdown-report.ts index d7a7700..00b199c 100644 --- a/src/reporting/render-markdown-report.ts +++ b/src/reporting/render-markdown-report.ts @@ -49,6 +49,20 @@ function appendRuntimeScorecard(lines: string[], result: CheckResult) { lines.push(`| Protocol headers | ${remote.protocolHeaders.toUpperCase()} |`); lines.push(`| Authorization | ${remote.authorization.toUpperCase()} |`); lines.push(`| Overall | ${remote.overall.toUpperCase()} |`); + + if (remote.reliability) { + const reliability = remote.reliability; + lines.push("", "## Remote Transport Reliability", ""); + lines.push("| Check | Status |"); + lines.push("| --- | --- |"); + lines.push(`| GET SSE | ${reliability.getSse.toUpperCase()} |`); + lines.push(`| Session propagation | ${reliability.sessionPropagation.toUpperCase()} |`); + lines.push(`| Resumability | ${reliability.resumability.toUpperCase()} |`); + lines.push(`| Disconnect safety | ${reliability.disconnectSafety.toUpperCase()} |`); + lines.push(`| Session restart | ${reliability.sessionRestart.toUpperCase()} |`); + lines.push(`| Termination | ${reliability.termination.toUpperCase()} |`); + lines.push(`| Overall | ${reliability.overall.toUpperCase()} |`); + } } } diff --git a/src/reporting/render-text-report.ts b/src/reporting/render-text-report.ts index e79153d..0e2b176 100644 --- a/src/reporting/render-text-report.ts +++ b/src/reporting/render-text-report.ts @@ -73,6 +73,18 @@ function appendRuntimeScorecard(lines: string[], result: CheckResult) { lines.push(`protocol headers: ${remote.protocolHeaders}`); lines.push(`authorization: ${remote.authorization}`); lines.push(`overall: ${remote.overall}`); + + if (remote.reliability) { + const reliability = remote.reliability; + lines.push("", "Remote Transport Reliability", "----------------------------"); + lines.push(`GET SSE: ${reliability.getSse}`); + lines.push(`Session propagation: ${reliability.sessionPropagation}`); + lines.push(`Resumability: ${reliability.resumability}`); + lines.push(`Disconnect safety: ${reliability.disconnectSafety}`); + lines.push(`Session restart: ${reliability.sessionRestart}`); + lines.push(`Termination: ${reliability.termination}`); + lines.push(`Overall: ${reliability.overall}`); + } } } diff --git a/src/rules/rule-catalog.ts b/src/rules/rule-catalog.ts index c0411b7..0feb38f 100644 --- a/src/rules/rule-catalog.ts +++ b/src/rules/rule-catalog.ts @@ -575,6 +575,114 @@ export const ruleCatalog: RuleDefinition[] = [ why: "Authorization readiness cannot be confirmed without bounded metadata responses.", fix: "Make protected-resource and authorization-server metadata available over HTTPS.", example: "Return application/json discovery metadata." + }, + { + id: "plugin.runtime.remote.reliability.get.status", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint rejected the SSE transport request.", + why: "Clients cannot rely on server-to-client streaming when an accepted SSE request does not return the expected status.", + fix: "Return HTTP 200 with a valid SSE response, or HTTP 405 when GET streaming is not supported.", + example: "HTTP/1.1 405 Method Not Allowed" + }, + { + id: "plugin.runtime.remote.reliability.get.content_type", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint returned a non-SSE media type for GET streaming.", + why: "Clients cannot interpret a successful streaming response unless it is declared as Server-Sent Events.", + fix: "Return Content-Type: text/event-stream for accepted GET streaming requests.", + example: "Content-Type: text/event-stream" + }, + { + id: "plugin.runtime.remote.reliability.get.inconclusive", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP GET streaming check was inconclusive.", + why: "A bounded probe could not observe a complete SSE event, so streaming reliability remains uncertain.", + fix: "Emit complete SSE event frames promptly when server-to-client streaming is supported.", + example: "id: event-1\n\ndata: {}\n\n" + }, + { + id: "plugin.runtime.remote.reliability.get.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP GET streaming request could not be completed.", + why: "Clients cannot rely on server-to-client streaming when a bounded transport request fails.", + fix: "Keep the Streamable HTTP transport reachable within the configured request bounds.", + example: "Complete the GET request before the runtime timeout." + }, + { + id: "plugin.runtime.remote.reliability.get.malformed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint emitted malformed SSE framing.", + why: "Malformed event framing prevents clients from safely processing stream progress or reconnecting.", + fix: "Return complete SSE events with valid id and retry fields.", + example: "id: event-1\n\nevent: message\n\ndata: {}\n\n" + }, + { + id: "plugin.runtime.remote.reliability.resume.status", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint rejected the SSE resume request.", + why: "Clients cannot resume a dropped stream when the reconnect response has an unexpected status.", + fix: "Return HTTP 200 for an accepted Last-Event-ID reconnect request.", + example: "HTTP/1.1 200 OK" + }, + { + id: "plugin.runtime.remote.reliability.resume.content_type", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint returned a non-SSE media type for a resume request.", + why: "Clients cannot process a resumed stream unless it is declared as Server-Sent Events.", + fix: "Return Content-Type: text/event-stream after accepting a Last-Event-ID reconnect request.", + example: "Content-Type: text/event-stream" + }, + { + id: "plugin.runtime.remote.reliability.resume.inconclusive", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP SSE resume check was inconclusive.", + why: "A bounded probe could not confirm a complete resumed event, so reconnect reliability remains uncertain.", + fix: "Emit complete SSE event frames after accepting a reconnect.", + example: "Return one complete SSE event after a Last-Event-ID request." + }, + { + id: "plugin.runtime.remote.reliability.resume.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP SSE resume request could not be completed.", + why: "Clients cannot recover from a dropped stream when a bounded reconnect fails.", + fix: "Accept one bounded SSE reconnect using Last-Event-ID.", + example: "Honor the Last-Event-ID header on one reconnect." + }, + { + id: "plugin.runtime.remote.reliability.resume.malformed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint emitted malformed SSE framing after reconnecting.", + why: "Malformed resumed events prevent clients from safely continuing stream processing.", + fix: "Return complete SSE events with valid id and retry fields after reconnecting.", + example: "Return a complete event frame after reconnecting." + }, + { + id: "plugin.runtime.remote.reliability.session_restart.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP session could not be restarted.", + why: "Clients cannot recover from an expired MCP session when a fresh initialization sequence fails.", + fix: "Accept a fresh initialize sequence after an expired MCP session.", + example: "Return a valid initialize response after HTTP 404 for a session-bound request." + }, + { + id: "plugin.runtime.remote.reliability.termination.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP session could not be terminated.", + why: "A server that advertises session lifecycle support must safely handle the approved termination request.", + fix: "Return a successful response or HTTP 405 for a bounded MCP session DELETE request.", + example: "HTTP/1.1 204 No Content" } ]; diff --git a/tests/json-runtime-scorecard.test.ts b/tests/json-runtime-scorecard.test.ts index 4673daf..cfe8bcd 100644 --- a/tests/json-runtime-scorecard.test.ts +++ b/tests/json-runtime-scorecard.test.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { runCheck } from "../src/index.js"; +import { buildDoctorOutputContract } from "../src/core/output-contract.js"; import { buildJsonReport } from "../src/reporting/render-json-report.js"; describe("runtime scorecard", () => { @@ -95,4 +96,62 @@ describe("runtime scorecard", () => { overall: "warn" }); }); + + it("adds a complete remote reliability scorecard without changing the prior remote shape", () => { + const reliability = { + getSse: "pass" as const, + sessionPropagation: "pass" as const, + resumability: "skipped" as const, + disconnectSafety: "skipped" as const, + sessionRestart: "skipped" as const, + termination: "skipped" as const, + overall: "pass" as const + }; + const report = buildJsonReport( + { + targetPath: "/test/plugin", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + initialize: "skipped", + toolsList: "unsupported", + toolsCall: "unsupported", + resourcesList: "unsupported", + resourceRead: "unsupported", + resourceTemplatesList: "unsupported", + promptsList: "unsupported", + promptGet: "unsupported", + remote: { + transport: "pass", + networkSafety: "pass", + initialize: "pass", + contentType: "pass", + session: "absent", + protocolHeaders: "pass", + authorization: "skipped", + reliability, + overall: "pass" + } + } + }, + { runtimeProbeEnabled: true } + ); + + expect(report.summary.runtimeScorecard?.remote?.reliability).toEqual(reliability); + }); + + it("keeps reliability optional in the public contract while requiring a complete object when present", () => { + const contract = buildDoctorOutputContract("2026-07-26T00:00:00.000Z"); + const schema = contract.schemas.find((entry) => entry.id === "doctor.check.json")?.schema; + const remote = (schema?.properties as { summary?: { properties?: { runtimeScorecard?: { properties?: { remote?: { properties?: Record; required?: string[] } } } } } }) + .summary?.properties?.runtimeScorecard?.properties?.remote; + const reliability = remote?.properties?.reliability as { required?: string[]; additionalProperties?: boolean } | undefined; + + expect(remote?.required).not.toContain("reliability"); + expect(reliability).toMatchObject({ + required: ["getSse", "sessionPropagation", "resumability", "disconnectSafety", "sessionRestart", "termination", "overall"], + additionalProperties: false + }); + }); }); diff --git a/tests/markdown-report.test.ts b/tests/markdown-report.test.ts index 0aa1fff..86d7f65 100644 --- a/tests/markdown-report.test.ts +++ b/tests/markdown-report.test.ts @@ -100,6 +100,27 @@ describe("buildMarkdownReport", () => { expect(report).toContain("| Session | PRESENT-VALID |"); }); + it("renders remote reliability capability labels and statuses only", () => { + const report = buildMarkdownReport({ + targetPath: "example", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + ...runtimeScorecard, + remote: { + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "present-valid", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "warn", sessionRestart: "skipped", termination: "skipped", overall: "warn" } + } + } + }, { runtimeProbeEnabled: true }); + + expect(report).toContain("## Remote Transport Reliability"); + expect(report).toContain("| GET SSE | PASS |"); + expect(report).toContain("| Disconnect safety | WARN |"); + expect(report).not.toMatch(/session-secret-sentinel|event-secret-sentinel|retry-secret-sentinel|body-secret-sentinel/); + }); + it("renders a CI-friendly markdown summary", async () => { const targetPath = path.resolve("tests/fixtures/heuristic-long-plugin-description"); const result = await runCheck(targetPath); diff --git a/tests/remote-mcp-probe.test.ts b/tests/remote-mcp-probe.test.ts index 122f290..ee4e7f7 100644 --- a/tests/remote-mcp-probe.test.ts +++ b/tests/remote-mcp-probe.test.ts @@ -1,20 +1,28 @@ import { createServer, type Server, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import os from "node:os"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { probeRemoteMcpServer } from "../src/core/remote-mcp-probe.js"; +import { probeRuntimeConfig } from "../src/core/runtime-probe.js"; import type { BoundedHttpResponse } from "../src/core/bounded-http-client.js"; import type { RemoteLookup } from "../src/core/remote-network-policy.js"; import { packageVersion } from "../src/version.js"; const servers: Server[] = []; const openResponses: ServerResponse[] = []; +const temporaryDirectories: string[] = []; afterEach(async () => { openResponses.splice(0).forEach((response) => response.destroy()); - await Promise.all(servers.splice(0).map((server) => new Promise((resolve, reject) => { - server.close((error) => error ? reject(error) : resolve()); - }))); + await Promise.all([ + ...servers.splice(0).map((server) => new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + })), + ...temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })) + ]); }); async function startServer(handler: Parameters[0]): Promise { @@ -101,6 +109,15 @@ describe("probeRemoteMcpServer", () => { session: "present-valid", protocolHeaders: "pass", authorization: "skipped", + reliability: { + getSse: "pass", + sessionPropagation: "pass", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "pass" + }, overall: "pass" }); expect(requests).toHaveLength(3); @@ -129,6 +146,61 @@ describe("probeRemoteMcpServer", () => { assertPrivate(result); }); + it("merges reliability scorecards across remote servers field by field", async () => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(request.url === "/failing" ? 500 : 405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + const rootPath = await mkdtemp(path.join(os.tmpdir(), "codex-plugin-doctor-remote-merge-")); + temporaryDirectories.push(rootPath); + await writeFile(path.join(rootPath, ".mcp.json"), JSON.stringify({ + mcpServers: { + compliant: { url: `http://localhost:${port}/compliant` }, + failing: { url: `http://localhost:${port}/failing` } + } + }), "utf8"); + + const result = await probeRuntimeConfig(rootPath, ".mcp.json", { + allowNetwork: true, + allowLocalNetwork: true, + remoteLookup: localLookup(), + remoteRequestTimeoutMs: 100 + }); + + expect(result.scorecard.remote?.reliability).toEqual({ + getSse: "fail", + sessionPropagation: "skipped", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "fail" + }); + expect(result.findings).toEqual([ + expect.objectContaining({ + id: "plugin.runtime.remote.reliability.get.status", + severity: "fail", + evidence: expect.objectContaining({ serverName: "failing" }) + }) + ]); + }); + it("treats a bounded SSE GET 405 as protocol-compliant after initialization", async () => { const requests: Array<{ method: string; headers: Record; body: string }> = []; const port = await startServer((request, response) => { @@ -161,7 +233,7 @@ describe("probeRemoteMcpServer", () => { expect(requests[2]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); }); - it("keeps reliability failures internal until the public scorecard is wired", async () => { + it("publishes redacted reliability failures through the remote scorecard and findings", async () => { const port = await startServer((request, response) => { let body = ""; request.setEncoding("utf8"); @@ -185,12 +257,11 @@ describe("probeRemoteMcpServer", () => { const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); - expect(result.findings).toEqual([]); - expect(result.scorecard.overall).toBe("pass"); - expect(result.reliability.findings).toEqual([ + expect(result.findings).toEqual([ expect.objectContaining({ id: "plugin.runtime.remote.reliability.get.status", severity: "fail" }) ]); - expect(result.reliability.scorecard.overall).toBe("fail"); + expect(result.scorecard.overall).toBe("fail"); + expect(result.scorecard.reliability?.overall).toBe("fail"); assertPrivate(result); }); @@ -250,7 +321,7 @@ describe("probeRemoteMcpServer", () => { }); expect(result.findings).toEqual([]); - expect(result.reliability.scorecard).toMatchObject({ sessionRestart: "pass", resumability: "pass", termination: "pass", overall: "pass" }); + expect(result.scorecard.reliability).toMatchObject({ sessionRestart: "pass", resumability: "pass", termination: "pass", overall: "pass" }); expect(requests.map((request) => request.method)).toEqual([ "POST", "POST", "GET", "POST", "POST", "GET", "GET", "DELETE" ]); diff --git a/tests/render-text-report.test.ts b/tests/render-text-report.test.ts index b191bcb..a3c6e65 100644 --- a/tests/render-text-report.test.ts +++ b/tests/render-text-report.test.ts @@ -93,6 +93,27 @@ describe("renderTextReport", () => { expect(output).toContain("session: present-valid"); }); + it("renders remote reliability capability labels and statuses only", () => { + const output = renderTextReport({ + targetPath: "example", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + ...runtimeScorecard, + remote: { + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "present-valid", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "warn", sessionRestart: "skipped", termination: "skipped", overall: "warn" } + } + } + }); + + expect(output).toContain("Remote Transport Reliability"); + expect(output).toContain("GET SSE: pass"); + expect(output).toContain("Disconnect safety: warn"); + expect(output).not.toMatch(/session-secret-sentinel|event-secret-sentinel|retry-secret-sentinel|body-secret-sentinel/); + }); + it("renders a rich unicode summary for warn results", async () => { const result = await runCheck( path.resolve("tests/fixtures/heuristic-long-plugin-description") diff --git a/tests/rule-catalog.test.ts b/tests/rule-catalog.test.ts index 575bdbe..ea94b6e 100644 --- a/tests/rule-catalog.test.ts +++ b/tests/rule-catalog.test.ts @@ -104,6 +104,21 @@ const remoteRuntimeRules = [ "plugin.runtime.remote.authorization.metadata.unavailable" ] as const; +const remoteReliabilityRules = [ + "plugin.runtime.remote.reliability.get.status", + "plugin.runtime.remote.reliability.get.content_type", + "plugin.runtime.remote.reliability.get.inconclusive", + "plugin.runtime.remote.reliability.get.failed", + "plugin.runtime.remote.reliability.get.malformed", + "plugin.runtime.remote.reliability.resume.status", + "plugin.runtime.remote.reliability.resume.content_type", + "plugin.runtime.remote.reliability.resume.inconclusive", + "plugin.runtime.remote.reliability.resume.failed", + "plugin.runtime.remote.reliability.resume.malformed", + "plugin.runtime.remote.reliability.session_restart.failed", + "plugin.runtime.remote.reliability.termination.failed" +] as const; + describe("MCP 2025-11 conformance rule catalog", () => { it("resolves every evaluator finding with its public remediation contract", () => { expect(ruleCatalog.filter((rule) => rule.id.startsWith("mcp.conformance."))).toEqual( @@ -126,4 +141,10 @@ describe("MCP 2025-11 conformance rule catalog", () => { expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "fail" }); } }); + + it("resolves every emitted remote reliability finding with a fail remediation contract", () => { + for (const id of remoteReliabilityRules) { + expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "fail" }); + } + }); }); From 3de691b1ff3cf87cf696363980982f66c1c81453 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 15:12:29 +0300 Subject: [PATCH 07/15] fix: tighten remote reliability status contract --- src/core/output-contract.ts | 22 ++++++++++++---------- src/domain/types.ts | 16 +++++++++------- tests/json-runtime-scorecard.test.ts | 25 +++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/core/output-contract.ts b/src/core/output-contract.ts index 82dc1b5..c5daeac 100644 --- a/src/core/output-contract.ts +++ b/src/core/output-contract.ts @@ -50,6 +50,11 @@ const runtimeCapabilityStatusSchema = { enum: ["pass", "fail", "warn", "skipped", "unsupported"] }; +const remoteTransportReliabilityStatusSchema = { + type: "string", + enum: ["pass", "warn", "fail", "skipped"] +}; + const runtimeConformanceSchema = { type: "object", properties: { @@ -84,16 +89,13 @@ const runtimeConformanceSchema = { const remoteTransportReliabilitySchema = { type: "object", properties: { - getSse: runtimeCapabilityStatusSchema, - sessionPropagation: runtimeCapabilityStatusSchema, - resumability: runtimeCapabilityStatusSchema, - disconnectSafety: runtimeCapabilityStatusSchema, - sessionRestart: runtimeCapabilityStatusSchema, - termination: runtimeCapabilityStatusSchema, - overall: { - type: "string", - enum: ["pass", "warn", "fail", "skipped"] - } + getSse: remoteTransportReliabilityStatusSchema, + sessionPropagation: remoteTransportReliabilityStatusSchema, + resumability: remoteTransportReliabilityStatusSchema, + disconnectSafety: remoteTransportReliabilityStatusSchema, + sessionRestart: remoteTransportReliabilityStatusSchema, + termination: remoteTransportReliabilityStatusSchema, + overall: remoteTransportReliabilityStatusSchema }, required: [ "getSse", diff --git a/src/domain/types.ts b/src/domain/types.ts index c8be2ed..b1713ea 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -128,14 +128,16 @@ export interface RemoteRuntimeScorecard { overall: "pass" | "warn" | "fail" | "skipped"; } +export type RemoteTransportReliabilityStatus = "pass" | "warn" | "fail" | "skipped"; + export interface RemoteTransportReliabilityScorecard { - getSse: RuntimeCapabilityStatus; - sessionPropagation: RuntimeCapabilityStatus; - resumability: RuntimeCapabilityStatus; - disconnectSafety: RuntimeCapabilityStatus; - sessionRestart: RuntimeCapabilityStatus; - termination: RuntimeCapabilityStatus; - overall: "pass" | "warn" | "fail" | "skipped"; + getSse: RemoteTransportReliabilityStatus; + sessionPropagation: RemoteTransportReliabilityStatus; + resumability: RemoteTransportReliabilityStatus; + disconnectSafety: RemoteTransportReliabilityStatus; + sessionRestart: RemoteTransportReliabilityStatus; + termination: RemoteTransportReliabilityStatus; + overall: RemoteTransportReliabilityStatus; } export type McpConformanceProfile = diff --git a/tests/json-runtime-scorecard.test.ts b/tests/json-runtime-scorecard.test.ts index cfe8bcd..21c659f 100644 --- a/tests/json-runtime-scorecard.test.ts +++ b/tests/json-runtime-scorecard.test.ts @@ -141,17 +141,38 @@ describe("runtime scorecard", () => { expect(report.summary.runtimeScorecard?.remote?.reliability).toEqual(reliability); }); - it("keeps reliability optional in the public contract while requiring a complete object when present", () => { + it("keeps reliability optional while rejecting unsupported capability statuses", () => { const contract = buildDoctorOutputContract("2026-07-26T00:00:00.000Z"); const schema = contract.schemas.find((entry) => entry.id === "doctor.check.json")?.schema; const remote = (schema?.properties as { summary?: { properties?: { runtimeScorecard?: { properties?: { remote?: { properties?: Record; required?: string[] } } } } } }) .summary?.properties?.runtimeScorecard?.properties?.remote; - const reliability = remote?.properties?.reliability as { required?: string[]; additionalProperties?: boolean } | undefined; + const reliability = remote?.properties?.reliability as { + properties?: Record; + required?: string[]; + additionalProperties?: boolean; + } | undefined; expect(remote?.required).not.toContain("reliability"); expect(reliability).toMatchObject({ required: ["getSse", "sessionPropagation", "resumability", "disconnectSafety", "sessionRestart", "termination", "overall"], additionalProperties: false }); + for (const capability of [ + "getSse", + "sessionPropagation", + "resumability", + "disconnectSafety", + "sessionRestart", + "termination", + "overall" + ]) { + expect(reliability?.properties?.[capability]?.enum).not.toContain("unsupported"); + expect(reliability?.properties?.[capability]?.enum).toEqual([ + "pass", + "warn", + "fail", + "skipped" + ]); + } }); }); From 5a0b8f987e0d6da832d77bc6caac1521b17e6963 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 15:25:58 +0300 Subject: [PATCH 08/15] fix: align reliability rule severities --- src/rules/rule-catalog.ts | 4 ++-- tests/remote-transport-reliability.test.ts | 16 ++++++++++++++++ tests/rule-catalog.test.ts | 19 ++++++++++++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/rules/rule-catalog.ts b/src/rules/rule-catalog.ts index 0feb38f..18c7af3 100644 --- a/src/rules/rule-catalog.ts +++ b/src/rules/rule-catalog.ts @@ -597,7 +597,7 @@ export const ruleCatalog: RuleDefinition[] = [ { id: "plugin.runtime.remote.reliability.get.inconclusive", category: "runtime", - defaultSeverity: "fail", + defaultSeverity: "warn", summary: "The remote MCP GET streaming check was inconclusive.", why: "A bounded probe could not observe a complete SSE event, so streaming reliability remains uncertain.", fix: "Emit complete SSE event frames promptly when server-to-client streaming is supported.", @@ -642,7 +642,7 @@ export const ruleCatalog: RuleDefinition[] = [ { id: "plugin.runtime.remote.reliability.resume.inconclusive", category: "runtime", - defaultSeverity: "fail", + defaultSeverity: "warn", summary: "The remote MCP SSE resume check was inconclusive.", why: "A bounded probe could not confirm a complete resumed event, so reconnect reliability remains uncertain.", fix: "Emit complete SSE event frames after accepting a reconnect.", diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index b281408..540981c 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -155,6 +155,22 @@ describe("probeRemoteTransportReliability", () => { expect(result.scorecard.overall).toBe("warn"); }); + it("treats a bounded SSE resume timeout as inconclusive", async () => { + const timeout = new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "timeout", 200, { "content-type": "text/event-stream" }); + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\n\n"), + timeout + ]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.resume.inconclusive", severity: "warn" }) + ]); + expect(result.scorecard).toMatchObject({ resumability: "warn", overall: "warn" }); + assertRedacted(result); + }); + it("reinitializes once after a session-bound GET 404 and uses the replacement session", async () => { const fixture = scriptedRequest([response(404), response(405)]); let restarts = 0; diff --git a/tests/rule-catalog.test.ts b/tests/rule-catalog.test.ts index ea94b6e..fcc467d 100644 --- a/tests/rule-catalog.test.ts +++ b/tests/rule-catalog.test.ts @@ -104,21 +104,24 @@ const remoteRuntimeRules = [ "plugin.runtime.remote.authorization.metadata.unavailable" ] as const; -const remoteReliabilityRules = [ +const remoteReliabilityFailRules = [ "plugin.runtime.remote.reliability.get.status", "plugin.runtime.remote.reliability.get.content_type", - "plugin.runtime.remote.reliability.get.inconclusive", "plugin.runtime.remote.reliability.get.failed", "plugin.runtime.remote.reliability.get.malformed", "plugin.runtime.remote.reliability.resume.status", "plugin.runtime.remote.reliability.resume.content_type", - "plugin.runtime.remote.reliability.resume.inconclusive", "plugin.runtime.remote.reliability.resume.failed", "plugin.runtime.remote.reliability.resume.malformed", "plugin.runtime.remote.reliability.session_restart.failed", "plugin.runtime.remote.reliability.termination.failed" ] as const; +const remoteReliabilityWarnRules = [ + "plugin.runtime.remote.reliability.get.inconclusive", + "plugin.runtime.remote.reliability.resume.inconclusive" +] as const; + describe("MCP 2025-11 conformance rule catalog", () => { it("resolves every evaluator finding with its public remediation contract", () => { expect(ruleCatalog.filter((rule) => rule.id.startsWith("mcp.conformance."))).toEqual( @@ -142,9 +145,15 @@ describe("MCP 2025-11 conformance rule catalog", () => { } }); - it("resolves every emitted remote reliability finding with a fail remediation contract", () => { - for (const id of remoteReliabilityRules) { + it("resolves failing remote reliability findings with fail remediation contracts", () => { + for (const id of remoteReliabilityFailRules) { expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "fail" }); } }); + + it("resolves inconclusive remote reliability findings with warn remediation contracts", () => { + for (const id of remoteReliabilityWarnRules) { + expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "warn" }); + } + }); }); From 49e567c7d3a94d6bea9fc6c27acf6580072ce1c0 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 15:49:36 +0300 Subject: [PATCH 09/15] feat: gate remote transport reliability --- action.yml | 18 +++++ src/core/release-check.ts | 4 + src/core/release-evidence.ts | 4 + src/core/runtime-probe.ts | 7 ++ src/core/validate-plugin.ts | 7 +- src/domain/types.ts | 2 + src/mcp/generic-mcp-doctor.ts | 11 ++- src/run-cli.ts | 103 +++++++++++++++++++++++-- tests/action-metadata.test.ts | 11 +++ tests/check-command.test.ts | 35 +++++++++ tests/cli-command.test.ts | 52 +++++++++++++ tests/mcp-command.test.ts | 55 ++++++++++++- tests/release-check-command.test.ts | 23 ++++++ tests/release-evidence-command.test.ts | 27 +++++++ 14 files changed, 343 insertions(+), 16 deletions(-) diff --git a/action.yml b/action.yml index 38eca5c..4e6d647 100644 --- a/action.yml +++ b/action.yml @@ -22,6 +22,14 @@ inputs: description: Explicitly allow remote MCP runtime probes to contact loopback endpoints only (localhost, 127.0.0.0/8, or ::1). Private, link-local, multicast, unspecified, reserved, and NAT64 ranges remain blocked. required: false default: "false" + allow-session-lifecycle: + description: Explicitly allow remote MCP runtime probing to send one state-changing session termination request when a valid session is issued. + required: false + default: "false" + require-remote-reliability: + description: Fail the validation result unless every attempted remote MCP reliability scorecard passes; this does not grant network access. + required: false + default: "false" installed: description: Validate plugins from the local Codex plugin cache. required: false @@ -170,6 +178,8 @@ runs: env: ALLOW_NETWORK_INPUT: ${{ inputs['allow-network'] }} ALLOW_LOCAL_NETWORK_INPUT: ${{ inputs['allow-local-network'] }} + ALLOW_SESSION_LIFECYCLE_INPUT: ${{ inputs['allow-session-lifecycle'] }} + REQUIRE_REMOTE_RELIABILITY_INPUT: ${{ inputs['require-remote-reliability'] }} CORPUS_METRICS_MANIFEST_INPUT: ${{ inputs['corpus-metrics-manifest'] }} CORPUS_METRICS_BASELINE_INPUT: ${{ inputs['corpus-metrics-baseline'] }} CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT: ${{ inputs['corpus-metrics-fail-on-regression'] }} @@ -217,6 +227,14 @@ runs: args+=(--allow-local-network) fi + if [[ "$ALLOW_SESSION_LIFECYCLE_INPUT" == "true" ]]; then + args+=(--allow-session-lifecycle) + fi + + if [[ "$REQUIRE_REMOTE_RELIABILITY_INPUT" == "true" ]]; then + args+=(--require-remote-reliability) + fi + if [[ -n "${{ inputs.config }}" ]]; then args+=(--config "${{ inputs.config }}") fi diff --git a/src/core/release-check.ts b/src/core/release-check.ts index 60fd61e..1f7b1e5 100644 --- a/src/core/release-check.ts +++ b/src/core/release-check.ts @@ -44,6 +44,8 @@ export interface BuildReleaseCheckOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeSandbox?: RuntimeSandboxMode; runCheck?: (targetPath: string, options: CheckOptions) => Promise; } @@ -177,6 +179,8 @@ export async function buildReleaseCheck( runtime: runtimeProbeEnabled, ...(options.allowNetwork ? { allowNetwork: true } : {}), ...(options.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(options.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(options.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(options.runtimeSandbox ? { runtimeSandbox: options.runtimeSandbox } : {}) }); const securityResult = await buildSecurityAudit(resolvedPath); diff --git a/src/core/release-evidence.ts b/src/core/release-evidence.ts index 71031f5..13542ff 100644 --- a/src/core/release-evidence.ts +++ b/src/core/release-evidence.ts @@ -161,6 +161,8 @@ export interface BuildDoctorReleaseEvidenceOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; sandbox?: RuntimeSandboxMode; environment?: CompatibilityEnvironment; runCheck?: (targetPath: string, options?: CheckOptions) => Promise; @@ -516,6 +518,8 @@ export async function buildDoctorReleaseEvidenceReport( runtime: true, ...(options.allowNetwork ? { allowNetwork: true } : {}), ...(options.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(options.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(options.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(options.sandbox ? { runtimeSandbox: options.sandbox } : {}) } : {}; diff --git a/src/core/runtime-probe.ts b/src/core/runtime-probe.ts index 69a516f..de5fc65 100644 --- a/src/core/runtime-probe.ts +++ b/src/core/runtime-probe.ts @@ -1993,11 +1993,17 @@ export interface RuntimeProbeOptions { transcript?: (line: string) => void; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; remoteRequestTimeoutMs?: number; remoteLookup?: RemoteLookup; remoteRequest?: RemoteMcpRequest; } +export function remoteReliabilityGatePassed(scorecard: RuntimeScorecard | undefined): boolean { + return scorecard !== undefined && + (scorecard.remote === undefined || scorecard.remote.reliability?.overall === "pass"); +} + export async function probeRuntimeConfig( rootPath: string, mcpConfigPath: string, @@ -2054,6 +2060,7 @@ export async function probeRuntimeConfig( const remote = await probeRemoteMcpServer(serverName, url, { allowNetwork: options.allowNetwork, allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle, requestTimeoutMs: options.remoteRequestTimeoutMs, lookup: options.remoteLookup, request: options.remoteRequest diff --git a/src/core/validate-plugin.ts b/src/core/validate-plugin.ts index 0cee031..82bb516 100644 --- a/src/core/validate-plugin.ts +++ b/src/core/validate-plugin.ts @@ -11,7 +11,7 @@ import type { import { withFindingFingerprints } from "../reporting/finding-fingerprint.js"; import { discoverPackage } from "./discover-package.js"; import { inspectRemoteMcpUrl } from "./remote-url-policy.js"; -import { probeRuntime } from "./runtime-probe.js"; +import { probeRuntime, remoteReliabilityGatePassed } from "./runtime-probe.js"; function buildFailure( id: string, @@ -791,7 +791,8 @@ export async function validatePlugin( sandbox: options.runtimeSandbox, transcript: options.runtimeTranscript, allowNetwork: options.allowNetwork, - allowLocalNetwork: options.allowLocalNetwork + allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle }) : null; const findings = [ @@ -805,7 +806,7 @@ export async function validatePlugin( ); const hasFailures = fingerprintedFindings.some( (finding) => finding.severity === "fail" - ); + ) || (options.requireRemoteReliability === true && !remoteReliabilityGatePassed(runtimeResult?.scorecard)); const hasWarnings = fingerprintedFindings.some( (finding) => finding.severity === "warn" ); diff --git a/src/domain/types.ts b/src/domain/types.ts index b1713ea..24db339 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -52,6 +52,8 @@ export interface CheckOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeTranscript?: (line: string) => void; runtimeStartupTimeoutMs?: number; runtimeSandbox?: RuntimeSandboxMode; diff --git a/src/mcp/generic-mcp-doctor.ts b/src/mcp/generic-mcp-doctor.ts index 1a18166..c140bcb 100644 --- a/src/mcp/generic-mcp-doctor.ts +++ b/src/mcp/generic-mcp-doctor.ts @@ -8,7 +8,7 @@ import { readMcpConfigPath } from "../compatibility/compatibility-matrix.js"; import { readJsonFile } from "../core/read-json-file.js"; -import { probeRuntimeConfig } from "../core/runtime-probe.js"; +import { probeRuntimeConfig, remoteReliabilityGatePassed } from "../core/runtime-probe.js"; import type { Finding, FindingEvidence, @@ -42,6 +42,8 @@ export interface GenericMcpDoctorOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeStartupTimeoutMs?: number; } @@ -289,14 +291,17 @@ export async function buildGenericMcpDoctor( ? await probeRuntimeConfig(canonicalRootPath, canonicalMcpConfigPath, { startupTimeoutMs: options.runtimeStartupTimeoutMs, allowNetwork: options.allowNetwork, - allowLocalNetwork: options.allowLocalNetwork + allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle }) : null; const fingerprintedFindings = withFindingFingerprints( [...staticFindings, ...(runtimeResult?.findings ?? [])], rootPath ); - const status = mergeReportStatus(fingerprintedFindings, security); + const status = options.requireRemoteReliability === true && !remoteReliabilityGatePassed(runtimeResult?.scorecard) + ? "fail" + : mergeReportStatus(fingerprintedFindings, security); return { targetPath: rootPath, diff --git a/src/run-cli.ts b/src/run-cli.ts index 6c78436..617ead8 100644 --- a/src/run-cli.ts +++ b/src/run-cli.ts @@ -324,9 +324,16 @@ function parseRuntimeSandbox( function parseRemoteNetworkFlags( flags: string[], runtime: boolean -): { allowNetwork: boolean; allowLocalNetwork: boolean } | CliUsageError { +): { + allowNetwork: boolean; + allowLocalNetwork: boolean; + allowSessionLifecycle: boolean; + requireRemoteReliability: boolean; +} | CliUsageError { let allowNetwork = false; let allowLocalNetwork = false; + let allowSessionLifecycle = false; + let requireRemoteReliability = false; for (let index = 0; index < flags.length; index += 1) { const flag = flags[index]; @@ -343,11 +350,36 @@ function parseRemoteNetworkFlags( return new CliUsageError("--allow-local-network does not accept a value."); } allowLocalNetwork = true; - } else if (flag?.startsWith("--allow-network=") || flag?.startsWith("--allow-local-network=")) { + } else if (flag === "--allow-session-lifecycle") { + if (allowSessionLifecycle) return new CliUsageError("Duplicate runtime remote flag: --allow-session-lifecycle."); + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return new CliUsageError("--allow-session-lifecycle does not accept a value."); + } + allowSessionLifecycle = true; + } else if (flag === "--require-remote-reliability") { + if (requireRemoteReliability) return new CliUsageError("Duplicate runtime remote flag: --require-remote-reliability."); + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return new CliUsageError("--require-remote-reliability does not accept a value."); + } + requireRemoteReliability = true; + } else if ( + flag?.startsWith("--allow-network=") || + flag?.startsWith("--allow-local-network=") || + flag?.startsWith("--allow-session-lifecycle=") || + flag?.startsWith("--require-remote-reliability=") + ) { return new CliUsageError(`${flag.split("=", 1)[0]} does not accept a value.`); } } + if (allowSessionLifecycle && !runtime) { + return new CliUsageError("--allow-session-lifecycle requires --runtime."); + } + + if (requireRemoteReliability && !runtime) { + return new CliUsageError("--require-remote-reliability requires --runtime and --allow-network."); + } + if ((allowNetwork || allowLocalNetwork) && !runtime) { return new CliUsageError("--allow-network requires --runtime."); } @@ -356,7 +388,15 @@ function parseRemoteNetworkFlags( return new CliUsageError("--allow-local-network requires --allow-network."); } - return { allowNetwork, allowLocalNetwork }; + if (allowSessionLifecycle && !allowNetwork) { + return new CliUsageError("--allow-session-lifecycle requires --allow-network."); + } + + if (requireRemoteReliability && !allowNetwork) { + return new CliUsageError("--require-remote-reliability requires --runtime and --allow-network."); + } + + return { allowNetwork, allowLocalNetwork, allowSessionLifecycle, requireRemoteReliability }; } function printUsage(io: CliIo): void { @@ -1185,6 +1225,8 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] runtime: boolean; allowNetwork: boolean; allowLocalNetwork: boolean; + allowSessionLifecycle: boolean; + requireRemoteReliability: boolean; } | string { if (!commandTarget || commandTarget.startsWith("--")) { return "Missing target path. Usage: codex-plugin-doctor mcp [--runtime] [--json] [--output ]"; @@ -1195,6 +1237,8 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] let runtime = false; let allowNetwork = false; let allowLocalNetwork = false; + let allowSessionLifecycle = false; + let requireRemoteReliability = false; for (let index = 0; index < flags.length; index += 1) { const flag = flags[index]; @@ -1229,6 +1273,31 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] continue; } + if (flag === "--allow-session-lifecycle") { + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return "--allow-session-lifecycle does not accept a value."; + } + allowSessionLifecycle = true; + continue; + } + + if (flag === "--require-remote-reliability") { + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return "--require-remote-reliability does not accept a value."; + } + requireRemoteReliability = true; + continue; + } + + if ( + flag.startsWith("--allow-network=") || + flag.startsWith("--allow-local-network=") || + flag.startsWith("--allow-session-lifecycle=") || + flag.startsWith("--require-remote-reliability=") + ) { + return `${flag.split("=", 1)[0]} does not accept a value.`; + } + if (flag === "--output") { if (outputPath !== null) { return "Duplicate MCP flag: --output."; @@ -1261,8 +1330,10 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] jsonOutput, outputPath, runtime, - allowNetwork, - allowLocalNetwork + allowNetwork: remoteNetwork.allowNetwork, + allowLocalNetwork: remoteNetwork.allowLocalNetwork, + allowSessionLifecycle, + requireRemoteReliability }; } @@ -1654,7 +1725,9 @@ export async function runCli( }, { runtime: parsedMcpArgs.runtime, allowNetwork: parsedMcpArgs.allowNetwork, - allowLocalNetwork: parsedMcpArgs.allowLocalNetwork + allowLocalNetwork: parsedMcpArgs.allowLocalNetwork, + allowSessionLifecycle: parsedMcpArgs.allowSessionLifecycle, + requireRemoteReliability: parsedMcpArgs.requireRemoteReliability }); const renderedReport = parsedMcpArgs.jsonOutput ? renderGenericMcpDoctorJson(report) @@ -2045,6 +2118,8 @@ export async function runCli( runtime, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { sandbox: runtimeSandbox } : {}), environment: { env: terminalContext.env, @@ -2240,6 +2315,8 @@ export async function runCli( runtime, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { sandbox: runtimeSandbox } : {}), environment: { env: terminalContext.env, @@ -3262,7 +3339,9 @@ export async function runCli( }, { runtime: parsedMcpArgs.runtime, allowNetwork: parsedMcpArgs.allowNetwork, - allowLocalNetwork: parsedMcpArgs.allowLocalNetwork + allowLocalNetwork: parsedMcpArgs.allowLocalNetwork, + allowSessionLifecycle: parsedMcpArgs.allowSessionLifecycle, + requireRemoteReliability: parsedMcpArgs.requireRemoteReliability }); const renderedReport = parsedMcpArgs.jsonOutput ? renderGenericMcpDoctorJson(report) @@ -3659,6 +3738,8 @@ export async function runCli( runtime: runtimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { runtimeSandbox } : {}), runCheck: options.runCheckImpl }); @@ -3932,6 +4013,8 @@ export async function runCli( runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), runtimeTranscript: effectiveRuntimeProbeEnabled && verboseRuntime ? (line) => io.writeStderr(line) @@ -4034,7 +4117,9 @@ export async function runCli( await runCheckImpl(pluginRoot, { runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), - ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}) + ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}) }), applyPolicyToDoctorConfig( applyCheckProfile(await loadDoctorConfig(pluginRoot, configPath), checkProfile), @@ -4070,6 +4155,8 @@ export async function runCli( runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { runtimeSandbox } : {}), ...(effectiveRuntimeProbeEnabled && verboseRuntime ? { runtimeTranscript: (line: string) => io.writeStderr(line) } diff --git a/tests/action-metadata.test.ts b/tests/action-metadata.test.ts index 4dc3cdb..9c1db07 100644 --- a/tests/action-metadata.test.ts +++ b/tests/action-metadata.test.ts @@ -102,6 +102,17 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).not.toContain('args+=(--allow-local-network "${{ inputs'); }); + it("forwards explicit lifecycle consent and strict reliability gating through every Action output", async () => { + const actionMetadata = await readFile("action.yml", "utf8"); + + expect(actionMetadata).toMatch(/allow-session-lifecycle:[\s\S]*?default: "false"/); + expect(actionMetadata).toMatch(/require-remote-reliability:[\s\S]*?default: "false"/); + expect(actionMetadata).toContain('ALLOW_SESSION_LIFECYCLE_INPUT: ${{ inputs[\'allow-session-lifecycle\'] }}'); + expect(actionMetadata).toContain('REQUIRE_REMOTE_RELIABILITY_INPUT: ${{ inputs[\'require-remote-reliability\'] }}'); + expect(actionMetadata).toContain('args+=(--allow-session-lifecycle)'); + expect(actionMetadata).toContain('args+=(--require-remote-reliability)'); + }); + it("documents loopback-only consent without permitting private or reserved ranges", async () => { const actionMetadata = await readFile("action.yml", "utf8"); const actionUsage = await readFile("docs/guides/github-action.md", "utf8"); diff --git a/tests/check-command.test.ts b/tests/check-command.test.ts index baf0abc..52b9105 100644 --- a/tests/check-command.test.ts +++ b/tests/check-command.test.ts @@ -4,6 +4,22 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { runCheck } from "../src/index.js"; +import { remoteReliabilityGatePassed } from "../src/core/runtime-probe.js"; +import type { RuntimeScorecard } from "../src/domain/types.js"; + +function runtimeScorecard(remote?: RuntimeScorecard["remote"]): RuntimeScorecard { + return { + initialize: "skipped", + toolsList: "skipped", + toolsCall: "skipped", + resourcesList: "skipped", + resourceRead: "skipped", + resourceTemplatesList: "skipped", + promptsList: "skipped", + promptGet: "skipped", + ...(remote ? { remote } : {}) + }; +} async function createPluginWithMissingSkillFiles(): Promise { const targetPath = await mkdtemp( @@ -28,6 +44,25 @@ async function createPluginWithMissingSkillFiles(): Promise { } describe("runCheck", () => { + it.each(["warn", "fail", "skipped"] as const)("rejects a %s remote reliability result at the strict gate", (overall) => { + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: overall, sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "pass", sessionRestart: "skipped", termination: "skipped", overall } + }))).toBe(false); + }); + + it("accepts a passing remote reliability result and ignores local-only runtime scorecards", () => { + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "pass", sessionRestart: "skipped", termination: "skipped", overall: "pass" } + }))).toBe(true); + expect(remoteReliabilityGatePassed(runtimeScorecard())).toBe(true); + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass" + }))).toBe(false); + expect(remoteReliabilityGatePassed(undefined)).toBe(false); + }); + it("distinguishes repeated skill findings with package-relative evidence", async () => { const targetPath = await createPluginWithMissingSkillFiles(); diff --git a/tests/cli-command.test.ts b/tests/cli-command.test.ts index 1dc7051..dd7ab0b 100644 --- a/tests/cli-command.test.ts +++ b/tests/cli-command.test.ts @@ -219,6 +219,10 @@ describe("runCli", () => { ["--runtme"], ["unexpected"], ["--runtime", "--runtime"], + ["--runtime", "--allow-network", "--allow-session-lifecycle", "--allow-session-lifecycle"], + ["--runtime", "--allow-network", "--require-remote-reliability", "--require-remote-reliability"], + ["--runtime", "--allow-network", "--allow-session-lifecycle", "true"], + ["--runtime", "--allow-network", "--require-remote-reliability", "true"], ["--json", "--json"], ["--output"], ["--output", "--json"], @@ -3015,6 +3019,54 @@ describe("runCli", () => { }); }); + it("propagates explicit lifecycle consent and strict remote reliability gating to a runtime check", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + const exitCode = await runCli( + [ + "check", + "tests/fixtures/valid-plugin", + "--runtime", + "--allow-network", + "--allow-session-lifecycle", + "--require-remote-reliability", + "--json" + ], + io, + { runCheckImpl } + ); + + expect(exitCode).toBe(0); + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + + it.each([ + [["--allow-session-lifecycle"], "--allow-session-lifecycle requires --runtime."], + [["--runtime", "--allow-session-lifecycle"], "--allow-session-lifecycle requires --allow-network."], + [["--runtime", "--require-remote-reliability"], "--require-remote-reliability requires --runtime and --allow-network."], + [["--runtime", "--allow-network", "--allow-session-lifecycle", "--allow-session-lifecycle"], "Duplicate runtime remote flag: --allow-session-lifecycle."], + [["--runtime", "--allow-network", "--require-remote-reliability", "--require-remote-reliability"], "Duplicate runtime remote flag: --require-remote-reliability."] + ])("rejects invalid lifecycle and reliability runtime flag combinations", async (flags, message) => { + const { io, stderr } = createIo(); + + const exitCode = await runCli(["check", "tests/fixtures/valid-plugin", ...flags], io); + + expect(exitCode).toBe(2); + expect(stderr.join("")).toContain(message); + }); + it("initializes a minimal Codex plugin package", async () => { const targetPath = await mkdtemp(path.join(os.tmpdir(), "codex-plugin-init-")); const { io, stdout, stderr } = createIo(); diff --git a/tests/mcp-command.test.ts b/tests/mcp-command.test.ts index f752707..85170de 100644 --- a/tests/mcp-command.test.ts +++ b/tests/mcp-command.test.ts @@ -36,7 +36,7 @@ async function createStandaloneMcpPackage(mcpConfig: unknown): Promise { return targetPath; } -async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = {}): Promise<{ +async function startRemoteMcpServer(options: { invalidInitialize?: boolean; session?: boolean; incompleteSse?: boolean } = {}): Promise<{ url: string; requests: string[]; close(): Promise; @@ -48,10 +48,22 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { request.on("end", () => { if (request.method === "GET") { requests.push("GET"); + if (options.incompleteSse) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("id: bounded-event\\n"); + return; + } response.writeHead(405); response.end(); return; } + + if (request.method === "DELETE") { + requests.push("DELETE"); + response.writeHead(204); + response.end(); + return; + } const message = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { method: string }; requests.push(message.method); @@ -62,7 +74,10 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { return; } - response.writeHead(200, { "content-type": "application/json" }); + response.writeHead(200, { + "content-type": "application/json", + ...(options.session ? { "mcp-session-id": "session-for-test" } : {}) + }); response.end(JSON.stringify({ jsonrpc: "2.0", id: 1, @@ -153,6 +168,42 @@ describe("mcp command", () => { } }); + it("requires lifecycle consent before terminating a remote session", async () => { + const remote = await startRemoteMcpServer({ session: true }); + const targetPath = await createStandaloneMcpPackage({ mcpServers: { local: { url: remote.url } } }); + const { io, stderr } = createIo(); + + try { + const exitCode = await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--allow-session-lifecycle", "--json" + ], io); + + expect(exitCode).toBe(0); + expect(stderr).toEqual([]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET", "DELETE"]); + } finally { + await remote.close(); + } + }); + + it("turns an inconclusive remote reliability scorecard into a blocking result only when requested", async () => { + const remote = await startRemoteMcpServer({ incompleteSse: true }); + const targetPath = await createStandaloneMcpPackage({ mcpServers: { local: { url: remote.url } } }); + const withoutGate = createIo(); + const withGate = createIo(); + + try { + expect(await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--json" + ], withoutGate.io)).toBe(0); + expect(await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--require-remote-reliability", "--json" + ], withGate.io)).toBe(1); + } finally { + await remote.close(); + } + }); + it("preserves the worst remote status when a later remote server passes", async () => { const failing = await startRemoteMcpServer({ invalidInitialize: true }); const passing = await startRemoteMcpServer(); diff --git a/tests/release-check-command.test.ts b/tests/release-check-command.test.ts index ace816b..e276187 100644 --- a/tests/release-check-command.test.ts +++ b/tests/release-check-command.test.ts @@ -88,6 +88,29 @@ describe("release-check command", () => { }); describe("CLI", () => { + it("forwards lifecycle consent and strict reliability gating to release validation", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + await runCli([ + "release", "check", "tests/fixtures/valid-plugin-with-mcp", "--runtime", "--allow-network", + "--allow-session-lifecycle", "--require-remote-reliability", "--json" + ], io, { runCheckImpl }); + + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + it("keeps runtime probing disabled by default", async () => { const { io, stdout, stderr } = createIo(); const runCheckImpl = vi.fn(async (targetPath: string) => ({ diff --git a/tests/release-evidence-command.test.ts b/tests/release-evidence-command.test.ts index 741425e..a1c7de5 100644 --- a/tests/release-evidence-command.test.ts +++ b/tests/release-evidence-command.test.ts @@ -25,6 +25,33 @@ function createIo() { } describe("doctor release-evidence command", () => { + it("forwards lifecycle consent and strict reliability gating to release evidence validation", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + await runCli([ + "doctor", "release-evidence", "examples/codex-doctor-runtime", "--sign-key-env", "DOCTOR_SIGNING_KEY", + "--allow-dirty", "--allow-untagged", "--runtime", "--allow-network", "--allow-session-lifecycle", + "--require-remote-reliability", "--json" + ], io, { + terminalContext: { env: { DOCTOR_SIGNING_KEY: "release-secret" }, platform: "win32" }, + runCheckImpl + }); + + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + it("renders a signed release evidence bundle as JSON", async () => { const { io, stdout, stderr } = createIo(); From 1759b8c16ed5f6135bd30aa664703e90d3c1eccf Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 16:03:52 +0300 Subject: [PATCH 10/15] fix: document remote reliability CLI gates --- src/run-cli.ts | 3 +++ tests/cli-command.test.ts | 13 +++++++++++++ tests/mcp-command.test.ts | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/run-cli.ts b/src/run-cli.ts index 617ead8..d059fdb 100644 --- a/src/run-cli.ts +++ b/src/run-cli.ts @@ -412,6 +412,9 @@ function printUsage(io: CliIo): void { io.writeStderr( "Baseline gating: codex-plugin-doctor baseline create --output [--runtime]\n codex-plugin-doctor check --baseline " ); + io.writeStderr( + "Remote MCP runtime flags (check, mcp, release check, and doctor release-evidence): --runtime --allow-network [--allow-local-network] [--allow-session-lifecycle] [--require-remote-reliability]\n --allow-session-lifecycle requires --runtime --allow-network and can terminate a remote session.\n --require-remote-reliability requires --runtime --allow-network, blocks non-pass reliability, and does not grant network consent." + ); } const suppressUsageText = [ diff --git a/tests/cli-command.test.ts b/tests/cli-command.test.ts index dd7ab0b..68a2ef4 100644 --- a/tests/cli-command.test.ts +++ b/tests/cli-command.test.ts @@ -3019,6 +3019,19 @@ describe("runCli", () => { }); }); + it("documents remote lifecycle consent and reliability gate dependencies in general help", async () => { + const { io, stderr } = createIo(); + + expect(await runCli([], io)).toBe(2); + + const usage = stderr.join(""); + expect(usage).toContain("--allow-session-lifecycle"); + expect(usage).toContain("--require-remote-reliability"); + expect(usage).toContain("can terminate a remote session"); + expect(usage).toContain("requires --runtime --allow-network"); + expect(usage).toContain("does not grant network consent"); + }); + it("propagates explicit lifecycle consent and strict remote reliability gating to a runtime check", async () => { const { io, stderr } = createIo(); const runCheckImpl = vi.fn(async (targetPath: string) => ({ diff --git a/tests/mcp-command.test.ts b/tests/mcp-command.test.ts index 85170de..33a0c5d 100644 --- a/tests/mcp-command.test.ts +++ b/tests/mcp-command.test.ts @@ -204,6 +204,34 @@ describe("mcp command", () => { } }); + it("blocks the strict gate when one of multiple remote reliability scorecards warns", async () => { + const passing = await startRemoteMcpServer(); + const warning = await startRemoteMcpServer({ incompleteSse: true }); + const targetPath = await createStandaloneMcpPackage({ + mcpServers: { + passing: { url: passing.url }, + warning: { url: warning.url } + } + }); + const { io, stdout, stderr } = createIo(); + + try { + const exitCode = await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--require-remote-reliability", "--json" + ], io); + const output = JSON.parse(stdout.join("")); + + expect(exitCode).toBe(1); + expect(stderr).toEqual([]); + expect(output.runtimeScorecard.remote.reliability.overall).toBe("warn"); + expect(passing.requests).toEqual(["initialize", "notifications/initialized", "GET"]); + expect(warning.requests).toEqual(["initialize", "notifications/initialized", "GET"]); + } finally { + await passing.close(); + await warning.close(); + } + }); + it("preserves the worst remote status when a later remote server passes", async () => { const failing = await startRemoteMcpServer({ invalidInitialize: true }); const passing = await startRemoteMcpServer(); From fd22c775dbd57e1b014c70ee2f449620aeb2e9e2 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 16:17:11 +0300 Subject: [PATCH 11/15] docs: document remote transport reliability --- README.md | 8 +- docs/README.md | 1 + docs/architecture/remote-mcp-readiness.md | 9 +- .../remote-mcp-transport-reliability.md | 211 ++++++++++++++++++ docs/guides/github-action.md | 3 +- docs/guides/release-gating.md | 10 + .../runtime-approval-and-sandboxing.md | 2 + src/core/doctor-export-bundle.ts | 4 +- tests/action-metadata.test.ts | 2 + tests/public-readiness.test.ts | 16 +- tests/release-evidence-command.test.ts | 22 ++ tests/remote-transport-reliability.test.ts | 58 +++++ 12 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 docs/architecture/remote-mcp-transport-reliability.md diff --git a/README.md b/README.md index 2687291..6b619e1 100644 --- a/README.md +++ b/README.md @@ -83,14 +83,18 @@ Runtime MCP validation with `--runtime`: ### Remote MCP Readiness -Remote MCP runtime probing is disabled until you explicitly pass `--allow-network`. Local endpoints also require `--allow-local-network`: +Remote MCP runtime probing is disabled until you explicitly pass `--runtime --allow-network`. Loopback endpoints also require `--allow-local-network`: ```bash codex-plugin-doctor check ./remote-mcp --runtime --allow-network codex-plugin-doctor check ./remote-mcp --runtime --allow-network --allow-local-network ``` -The probe makes only bounded, read-only protocol and OAuth metadata-discovery requests, redacts report output, and applies SSRF controls. It does not authenticate or follow redirects. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md). +The probe makes only bounded protocol and OAuth metadata-discovery requests, redacts report output, and applies SSRF controls. It does not authenticate or follow redirects. + +Remote transport reliability adds one bounded SSE GET after initialization. HTTP `200` must be `text/event-stream`; HTTP `405` is compliant when the endpoint does not offer server-to-client SSE. At most one SSE resume and one session restart are attempted, and no remote response content is retained. This is a bounded readiness check, not a live interoperability, delivery-guarantee, or load-test claim. + +`--allow-session-lifecycle` is disabled by default and is state-changing: only after a valid `MCP-Session-Id` it permits one bounded session `DELETE`. `--require-remote-reliability` is a strict result gate that requires a passing reliability scorecard; it grants no network consent, so `--runtime --allow-network` (and loopback consent when applicable) remain required. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](./docs/architecture/remote-mcp-transport-reliability.md). Output formats: diff --git a/docs/README.md b/docs/README.md index c1edf02..81704a9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ This directory contains public documentation for users, contributors, and securi - [Runtime Sandbox and External Corpus](architecture/runtime-sandbox-and-external-corpus.md) - [MCP 2025-11 Conformance](architecture/mcp-2025-11-conformance.md) - [Remote MCP Readiness](architecture/remote-mcp-readiness.md) +- [Remote MCP Transport Reliability](architecture/remote-mcp-transport-reliability.md) - [Real-World Corpus Quality Metrics](architecture/real-world-corpus-quality-metrics.md) - [Corpus Metrics Regression Diff](architecture/corpus-metrics-regression-diff.md) diff --git a/docs/architecture/remote-mcp-readiness.md b/docs/architecture/remote-mcp-readiness.md index d6ec781..c64fae5 100644 --- a/docs/architecture/remote-mcp-readiness.md +++ b/docs/architecture/remote-mcp-readiness.md @@ -17,7 +17,13 @@ The same consent is available in the GitHub Action through `allow-network: "true ## Read-Only Scope -The probe uses a bounded HTTP request for MCP initialization and only follows the OAuth metadata-discovery path advertised by an unauthenticated challenge. It never sends credentials or tokens, and reporting redacts sensitive values, response bodies, session identifiers, and authorization metadata. +The probe uses bounded HTTP requests for MCP initialization and only follows the OAuth metadata-discovery path advertised by an unauthenticated challenge. It never sends credentials or tokens, and reporting redacts sensitive values, response bodies, session identifiers, SSE event identifiers, SSE retry values, and authorization metadata. + +## Transport Reliability + +The optional reliability scorecard runs only within the same `--runtime --allow-network` consent boundary; loopback endpoints still need `--allow-local-network`. It makes one bounded GET request after initialization. A `200` response must use `text/event-stream`; a `405` response is compliant when server-to-client SSE is unsupported. The probe can make at most one SSE resume and one session restart, and it retains no raw response body or SSE data. It is a bounded readiness check, not a claim of live remote interoperability, replay correctness, delivery guarantees, or load capacity. + +`--allow-session-lifecycle` is false by default. It is state-changing and permits one bounded `DELETE` only after initialization supplies a valid `MCP-Session-Id`. `--require-remote-reliability` is a strict result gate: the command requires a passing reliability scorecard, but the flag grants no network or loopback consent. See [Remote MCP Transport Reliability](remote-mcp-transport-reliability.md) for the protocol sequence and classifications. ## SSRF Controls @@ -30,7 +36,6 @@ These checks reduce SSRF exposure but cannot account for every network topology. - authenticated OAuth - custom headers - remote tool/resource/prompt/task calls -- GET SSE/resumability - redirects Use a dedicated MCP client with its own authorization and network policy when any of these capabilities are required. diff --git a/docs/architecture/remote-mcp-transport-reliability.md b/docs/architecture/remote-mcp-transport-reliability.md new file mode 100644 index 0000000..1ee8823 --- /dev/null +++ b/docs/architecture/remote-mcp-transport-reliability.md @@ -0,0 +1,211 @@ +# Remote MCP Transport Reliability + +## Purpose + +Codex Plugin Doctor verifies whether an explicitly approved remote MCP endpoint +behaves safely and predictably across Streamable HTTP session and SSE lifecycle +boundaries. The probe is a bounded readiness check, not a general-purpose MCP +client or load test. + +## Goals + +- validate protocol-compliant GET behavior after initialization +- validate session propagation without exposing session identifiers +- observe bounded SSE event framing and resumability when evidence is available +- classify disconnects, timeouts, retry delays, and one session restart without + retaining remote response content +- optionally terminate an initialized session after explicit user consent +- expose a stable, additive reliability scorecard and CI gate + +## Non-Goals + +- authenticated OAuth requests +- tool, resource, prompt, or task execution +- arbitrary custom request headers +- unbounded SSE subscriptions +- automatic retries beyond one evidence-driven resume attempt or one + protocol-required session restart +- performance, availability, or load testing + +## Consent Model + +Remote probing continues to require `--runtime --allow-network`. Loopback +endpoints additionally require `--allow-local-network`. + +The default reliability probe does not send a session termination request. +`--allow-session-lifecycle` explicitly permits one bounded `DELETE` request +after the probe completes and only when initialization returned a valid +`MCP-Session-Id`. + +`--require-remote-reliability` is a result gate, not network consent. It is +invalid unless remote runtime probing is enabled. The command fails unless the +remote reliability scorecard reaches `pass`. + +## Probe Sequence + +1. Perform the existing bounded `initialize` request. +2. Validate `MCP-Session-Id` as visible ASCII when present. +3. Send `notifications/initialized` with the negotiated protocol and session + headers. +4. Send one bounded GET request with `Accept: text/event-stream`, the negotiated + protocol version, and the session identifier when present. +5. Accept HTTP 405 as a protocol-compliant declaration that the endpoint does + not offer a server-to-client SSE stream. +6. For HTTP 200, require `text/event-stream` and inspect only enough bytes to + identify a complete SSE event or reach the configured bounds. +7. If a valid SSE `id` field is observed, make one bounded reconnect attempt + using `Last-Event-ID`. Honor a valid server-provided SSE `retry` delay within + the remaining probe deadline. Do not reconnect when no event identifier is + observed. +8. If a subsequent request returns HTTP 404 for an issued session identifier, + discard that identifier and make one bounded re-initialization attempt + without it. Never restart a session more than once. +9. If lifecycle consent is enabled and a current session exists, send one bounded + `DELETE` request with the protocol and session headers. + +No step follows redirects. Every request reuses the existing DNS resolution, +peer matching, response-size, encoding, and timeout controls. +The complete probe has a fixed request-count ceiling; reconnect and session +restart paths cannot recurse. + +## HTTP Classification + +### GET + +- `200` with `text/event-stream`: supported; continue bounded SSE inspection +- `405`: compliant but unsupported; no resumability attempt +- any other status: fail +- `200` with another media type: fail + +An SSE connection that produces no complete event before the observation +deadline is inconclusive, not proof of malformed framing. A malformed complete +event, oversized response, unsafe peer change, or transport error fails the +relevant capability. + +A valid SSE `retry` field is respected before reconnecting. If its delay exceeds +the remaining bounded probe deadline, resumability is inconclusive and no late +request is sent. Invalid retry fields are ignored according to SSE parsing +rules. + +### Expired Session + +HTTP 404 on a request carrying an issued session identifier means the session +expired or was terminated. The probe discards the stale identifier and performs +one fresh initialize request without a session header. A successful restart +continues with the new session; a second session-expiry response fails restart +reliability. + +### DELETE + +- any `2xx`: termination passed +- `405`: compliant but unsupported +- any other status or transport failure: termination failed + +DELETE classification is `skipped` when lifecycle consent is absent or no +session identifier was issued. + +## Resumability + +The probe attempts resumability only when the first bounded GET produces a +complete event with a non-empty valid `id` field. The exact event identifier is +held in memory only for the duration of the reconnect request and is sent as +`Last-Event-ID`. + +A reconnect confirms request propagation and bounded transport behavior. It +does not claim delivery guarantees or replay correctness because the probe does +not retain or compare application payloads. + +## Scorecard + +The existing remote runtime scorecard gains an additive `reliability` object: + +```json +{ + "getSse": "pass", + "sessionPropagation": "pass", + "resumability": "skipped", + "disconnectSafety": "pass", + "sessionRestart": "skipped", + "termination": "skipped", + "overall": "pass" +} +``` + +Capability fields use the existing `pass`, `warn`, `fail`, and `skipped` +statuses. `overall` is: + +- `fail` when any attempted reliability check fails +- `warn` when no check fails but an attempted check is inconclusive +- `pass` when all applicable checks are compliant +- `skipped` when reliability probing does not run + +Protocol-compliant unsupported behavior, such as GET or DELETE returning 405, +does not lower an otherwise passing score. + +## Data Handling + +The following values must never appear in text, Markdown, JSON, evidence, +transcript, error, or debug output: + +- raw `MCP-Session-Id` values +- raw `Last-Event-ID` values +- raw SSE `retry` values +- SSE event data +- remote response bodies +- authorization metadata beyond existing redacted readiness classifications + +Reports expose only classifications and stable finding identifiers. The probe +does not write remote values to disk. + +## HTTP Client Changes + +The bounded HTTP client may add only the capabilities required by this design: + +- allow the `Last-Event-ID` request header +- preserve safe status and response-header metadata on a timeout after response + headers arrive +- keep existing three-second and one-megabyte upper bounds + +It must not add redirects, cookies, authorization headers, connection pooling, +or configurable unsafe headers. + +## CLI And GitHub Action + +The `check`, release-check, release-evidence, and GitHub Action paths that +already expose remote runtime probing receive consistent inputs: + +- `--allow-session-lifecycle` +- `--require-remote-reliability` +- `allow-session-lifecycle` +- `require-remote-reliability` + +Help output and public documentation must state that lifecycle consent can +change remote server state. Existing commands remain unchanged when both new +flags are absent. + +## Findings + +New findings use the `plugin.runtime.remote.reliability.*` namespace and must +provide actionable remediation without including remote values. At minimum, +tests cover invalid GET status, invalid SSE media type, malformed event framing, +resume failure, and termination failure. + +## Verification + +The implementation requires deterministic local HTTP fixtures for: + +- GET returning 405 +- valid SSE with no event identifier +- valid SSE with an event identifier and one resume request +- valid SSE retry delay and an over-deadline retry delay +- invalid SSE content type and malformed framing +- session propagation on initialized, GET, resume, and DELETE requests +- session expiry followed by one successful or failed re-initialization +- timeout and disconnect behavior +- DELETE disabled, successful, unsupported, and failed states +- output redaction across text, Markdown, JSON, and release evidence +- CLI validation and GitHub Action input forwarding +- `--require-remote-reliability` pass, warn, fail, and skipped outcomes + +Completion requires focused tests, the full test suite, build, dependency audit, +package verification, and a clean consumer installation. diff --git a/docs/guides/github-action.md b/docs/guides/github-action.md index 5d052b2..5d7e173 100644 --- a/docs/guides/github-action.md +++ b/docs/guides/github-action.md @@ -17,9 +17,10 @@ Remote MCP checks are off by default. Set `runtime: "true"` and give explicit ne runtime: "true" allow-network: "true" allow-local-network: "true" # Remove for public endpoints. + require-remote-reliability: "true" # Fails unless the bounded reliability scorecard passes. ``` -The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. Remote probes remain read-only and redact diagnostics; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) for SSRF and OAuth metadata-discovery boundaries. +The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. `require-remote-reliability` is a strict result gate, not network consent. Keep `allow-session-lifecycle: "false"` (the default) unless the workflow explicitly authorizes one bounded, state-changing session `DELETE` after a valid session is issued. Remote probes redact diagnostics and retain no raw remote content; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](../architecture/remote-mcp-transport-reliability.md) for SSRF, OAuth metadata-discovery, SSE, and lifecycle boundaries. ## Recommended Workflow diff --git a/docs/guides/release-gating.md b/docs/guides/release-gating.md index 9fce94a..1ba51cc 100644 --- a/docs/guides/release-gating.md +++ b/docs/guides/release-gating.md @@ -58,6 +58,16 @@ node dist/cli.js check ./path/to/plugin --json --runtime --output codex-plugin-d node dist/cli.js check ./path/to/plugin --json --runtime --sandbox docker --output codex-plugin-doctor-runtime-report.json ``` +### Remote Transport Reliability + +Remote MCP probing remains opt-in: it requires `--runtime --allow-network`, plus `--allow-local-network` for loopback endpoints. The reliability probe uses bounded requests, accepts HTTP `405` as compliant when GET SSE is unsupported, and makes no more than one resume or session restart. It does not establish live interoperability, delivery, or load-test guarantees. + +```bash +node dist/cli.js check ./path/to/plugin --runtime --allow-network --require-remote-reliability +``` + +`--require-remote-reliability` is a strict result gate and grants no network consent; the command passes only when the reliability scorecard passes. Keep `--allow-session-lifecycle` off for ordinary release gates. That opt-in is state-changing and permits one bounded session `DELETE` only when initialization supplied a valid session identifier. + Docker mode currently supports Node.js stdio MCP servers. It uses a read-only package mount and container filesystem, no network, an unprivileged user, dropped capabilities, bounded resources, and a limited writable `/tmp`. It fails closed and does not fall back to native execution. ### Release Readiness diff --git a/docs/security/runtime-approval-and-sandboxing.md b/docs/security/runtime-approval-and-sandboxing.md index b484096..dda7fab 100644 --- a/docs/security/runtime-approval-and-sandboxing.md +++ b/docs/security/runtime-approval-and-sandboxing.md @@ -10,6 +10,8 @@ Native probing preserves the existing host execution behavior: codex-plugin-doctor check ./plugin --runtime ``` +Remote MCP probing additionally requires `--allow-network`; loopback endpoints require `--allow-local-network`. Its transport reliability check is bounded and does not retain raw session IDs, event IDs, retry values, SSE data, or response bodies. `--allow-session-lifecycle` is false by default and is state-changing: it allows one bounded session `DELETE` only after a valid session identifier is issued. `--require-remote-reliability` is a strict pass gate, not network consent. + Docker probing is explicit and currently supports local Node.js stdio servers: ```bash diff --git a/src/core/doctor-export-bundle.ts b/src/core/doctor-export-bundle.ts index 15ec805..f415936 100644 --- a/src/core/doctor-export-bundle.ts +++ b/src/core/doctor-export-bundle.ts @@ -46,7 +46,9 @@ export function redactValue(value: unknown): unknown { return Object.fromEntries( Object.entries(value).map(([key, nestedValue]) => [ key, - redactValue(nestedValue) + /^(?:mcp[-_]?session[-_]?id|session[-_]?id|last[-_]?event[-_]?id|event[-_]?id|retry|(?:sse[-_]?)?data)$/i.test(key) + ? "[REDACTED_SECRET]" + : redactValue(nestedValue) ]) ); } diff --git a/tests/action-metadata.test.ts b/tests/action-metadata.test.ts index 9c1db07..af356f7 100644 --- a/tests/action-metadata.test.ts +++ b/tests/action-metadata.test.ts @@ -123,6 +123,8 @@ describe("GitHub Action metadata", () => { expect(document).toContain("loopback endpoints only"); expect(document).toContain("Private, link-local, multicast, unspecified, reserved, and NAT64 ranges remain blocked."); } + expect(actionUsage).toContain('require-remote-reliability: "true"'); + expect(actionUsage).toContain('allow-session-lifecycle: "false"'); }); it("documents the public GitHub Action consumer workflow", async () => { diff --git a/tests/public-readiness.test.ts b/tests/public-readiness.test.ts index dee991b..403aa7b 100644 --- a/tests/public-readiness.test.ts +++ b/tests/public-readiness.test.ts @@ -101,6 +101,9 @@ describe("public repository readiness", () => { const actionGuide = await readText("docs/guides/github-action.md"); const conformance = await readText("docs/architecture/mcp-2025-11-conformance.md"); const readiness = await readText("docs/architecture/remote-mcp-readiness.md"); + const reliability = await readText("docs/architecture/remote-mcp-transport-reliability.md"); + const releaseGating = await readText("docs/guides/release-gating.md"); + const runtimeSecurity = await readText("docs/security/runtime-approval-and-sandboxing.md"); const docsReadme = await readText("docs/README.md"); const security = await readText("docs/security/security-architecture.md"); expect(readme).toContain("Remote MCP Readiness"); @@ -113,9 +116,20 @@ describe("public repository readiness", () => { expect(readiness).toContain("authenticated OAuth"); expect(readiness).toContain("custom headers"); expect(readiness).toContain("remote tool/resource/prompt/task calls"); - expect(readiness).toContain("GET SSE/resumability"); + expect(readiness).toContain("one bounded GET request"); expect(readiness).toContain("redirects"); expect(docsReadme).toContain("Remote MCP Readiness"); + expect(docsReadme).toContain("Remote MCP Transport Reliability"); + expect(readme).toContain("--allow-session-lifecycle"); + expect(readme).toContain("--require-remote-reliability"); + expect(readiness).toContain("--allow-session-lifecycle"); + expect(readiness).toContain("--require-remote-reliability"); + expect(reliability).toContain("GET returning 405"); + expect(reliability).toContain("--runtime --allow-network"); + expect(reliability).toContain("one bounded `DELETE` request"); + expect(reliability).toContain("does not claim delivery guarantees"); + expect(releaseGating).toContain("--require-remote-reliability"); + expect(runtimeSecurity).toContain("--allow-session-lifecycle"); expect(security).toContain("runner or host egress controls"); expect(readiness).not.toMatch(/internal (implementation )?plan/i); }); diff --git a/tests/release-evidence-command.test.ts b/tests/release-evidence-command.test.ts index a1c7de5..17e4a26 100644 --- a/tests/release-evidence-command.test.ts +++ b/tests/release-evidence-command.test.ts @@ -457,6 +457,28 @@ describe("doctor release-evidence command", () => { expect(rendered).not.toContain("SHOULD_NOT_LEAK"); }); + it("never renders remote transport values in release evidence", () => { + const rendered = renderDoctorReleaseEvidenceJson({ + kind: "doctor.release.evidence", + schemaVersion: "1.0.0", + transport: { + sessionId: "release-session-canary-7d31", + lastEventId: "release-event-canary-91ac", + retry: "release-retry-canary-4f82", + data: "release-sse-data-canary-c8e5" + } + } as never); + + for (const canary of [ + "release-session-canary-7d31", + "release-event-canary-91ac", + "release-retry-canary-4f82", + "release-sse-data-canary-c8e5" + ]) { + expect(rendered).not.toContain(canary); + } + }); + it("verifies a release evidence bundle against its target package", async () => { const outputPath = path.join( await mkdtemp(path.join(os.tmpdir(), "codex-plugin-doctor-release-evidence-verify-")), diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index 540981c..a236cab 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { BoundedHttpError, type BoundedHttpRequestOptions, type BoundedHttpResponse } from "../src/core/bounded-http-client.js"; import { probeRemoteTransportReliability } from "../src/core/remote-transport-reliability.js"; +import { buildMarkdownReport } from "../src/reporting/render-markdown-report.js"; +import { renderTextReport } from "../src/reporting/render-text-report.js"; const endpoint = "https://mcp.example/mcp"; const protocolVersion = "2025-11-25"; @@ -58,6 +60,62 @@ function assertRedacted(value: unknown): void { } describe("probeRemoteTransportReliability", () => { + it("keeps remote transport canaries out of CLI and Action-facing report content", async () => { + const fixture = scriptedRequest([ + response( + 200, + "text/event-stream", + "id: report-event-canary-91ac\nretry: 917431\ndata: report-sse-data-canary-c8e5\n\n" + ) + ]); + const result = await probe(fixture.request, { + sessionId: "report-session-canary-7d31", + requestTimeoutMs: 10 + }); + const report = { + targetPath: "example", + status: "warn" as const, + exitCode: 0, + findings: result.findings, + runtimeScorecard: { + initialize: "pass" as const, + toolsList: "skipped" as const, + toolsCall: "skipped" as const, + resourcesList: "skipped" as const, + resourceRead: "skipped" as const, + resourceTemplatesList: "skipped" as const, + promptsList: "skipped" as const, + promptGet: "skipped" as const, + remote: { + transport: "pass" as const, + networkSafety: "pass" as const, + initialize: "pass" as const, + contentType: "pass" as const, + session: "present-valid" as const, + protocolHeaders: "pass" as const, + authorization: "skipped" as const, + overall: "warn" as const, + reliability: result.scorecard + } + } + }; + + for (const output of [ + JSON.stringify(report), + renderTextReport(report), + buildMarkdownReport(report, { runtimeProbeEnabled: true }) + ]) { + for (const canary of [ + "report-session-canary-7d31", + "report-event-canary-91ac", + "917431", + "report-sse-data-canary-c8e5" + ]) { + expect(output).not.toContain(canary); + } + } + }); + it("accepts GET 405 as compliant without attempting resume", async () => { const fixture = scriptedRequest([response(405)]); From bd726d5d009105e8743e5863df455a149ecdb303 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 16:28:37 +0300 Subject: [PATCH 12/15] fix: prevent remote session termination retries --- src/core/remote-transport-reliability.ts | 2 +- tests/remote-transport-reliability.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts index db2fb53..3aca5d8 100644 --- a/src/core/remote-transport-reliability.ts +++ b/src/core/remote-transport-reliability.ts @@ -175,7 +175,7 @@ export async function probeRemoteTransportReliability( return { error }; } - if (sessionId !== null && response.statusCode === 404) { + if (method === "GET" && sessionId !== null && response.statusCode === 404) { if (restarted) { scorecard.sessionRestart = "fail"; return { restartFailed: true }; diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index a236cab..fc30347 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -346,6 +346,28 @@ describe("probeRemoteTransportReliability", () => { assertRedacted(result); }); + it("treats a session-bound DELETE 404 as one failed termination without restart", async () => { + const fixture = scriptedRequest([response(405), response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + allowSessionLifecycle: true, + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(fixture.requests.map((request) => request.method)).toEqual(["GET", "DELETE"]); + expect(restarts).toBe(0); + expect(result.scorecard).toMatchObject({ termination: "fail", overall: "fail" }); + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.termination.failed", severity: "fail" }) + ]); + assertRedacted(result); + }); + it.each([ [false, response(405), "skipped", "pass"], [true, response(204), "pass", "pass"], From 12c1626800e88c23b3e4fdaf55464893fa221124 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 16:40:08 +0300 Subject: [PATCH 13/15] docs: clarify remote reliability gate semantics --- README.md | 2 +- docs/architecture/remote-mcp-readiness.md | 2 +- .../remote-mcp-transport-reliability.md | 17 +++++++++-------- docs/guides/github-action.md | 4 ++-- docs/guides/release-gating.md | 2 +- .../security/runtime-approval-and-sandboxing.md | 2 +- tests/public-readiness.test.ts | 6 ++++++ tests/remote-transport-reliability.test.ts | 3 ++- 8 files changed, 23 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6b619e1..41bf9a0 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ The probe makes only bounded protocol and OAuth metadata-discovery requests, red Remote transport reliability adds one bounded SSE GET after initialization. HTTP `200` must be `text/event-stream`; HTTP `405` is compliant when the endpoint does not offer server-to-client SSE. At most one SSE resume and one session restart are attempted, and no remote response content is retained. This is a bounded readiness check, not a live interoperability, delivery-guarantee, or load-test claim. -`--allow-session-lifecycle` is disabled by default and is state-changing: only after a valid `MCP-Session-Id` it permits one bounded session `DELETE`. `--require-remote-reliability` is a strict result gate that requires a passing reliability scorecard; it grants no network consent, so `--runtime --allow-network` (and loopback consent when applicable) remain required. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](./docs/architecture/remote-mcp-transport-reliability.md). +`--allow-session-lifecycle` is disabled by default and is state-changing: only after a valid `MCP-Session-Id` it permits one bounded session `DELETE`. `--require-remote-reliability` is a strict result gate: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. It grants no network consent, so `--runtime --allow-network` (and loopback consent when applicable) remain required. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](./docs/architecture/remote-mcp-transport-reliability.md). Output formats: diff --git a/docs/architecture/remote-mcp-readiness.md b/docs/architecture/remote-mcp-readiness.md index c64fae5..f7d199e 100644 --- a/docs/architecture/remote-mcp-readiness.md +++ b/docs/architecture/remote-mcp-readiness.md @@ -23,7 +23,7 @@ The probe uses bounded HTTP requests for MCP initialization and only follows the The optional reliability scorecard runs only within the same `--runtime --allow-network` consent boundary; loopback endpoints still need `--allow-local-network`. It makes one bounded GET request after initialization. A `200` response must use `text/event-stream`; a `405` response is compliant when server-to-client SSE is unsupported. The probe can make at most one SSE resume and one session restart, and it retains no raw response body or SSE data. It is a bounded readiness check, not a claim of live remote interoperability, replay correctness, delivery guarantees, or load capacity. -`--allow-session-lifecycle` is false by default. It is state-changing and permits one bounded `DELETE` only after initialization supplies a valid `MCP-Session-Id`. `--require-remote-reliability` is a strict result gate: the command requires a passing reliability scorecard, but the flag grants no network or loopback consent. See [Remote MCP Transport Reliability](remote-mcp-transport-reliability.md) for the protocol sequence and classifications. +`--allow-session-lifecycle` is false by default. It is state-changing and permits one bounded `DELETE` only after initialization supplies a valid `MCP-Session-Id`. `--require-remote-reliability` is a strict result gate: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. The flag grants no network or loopback consent. See [Remote MCP Transport Reliability](remote-mcp-transport-reliability.md) for the protocol sequence and classifications. ## SSRF Controls diff --git a/docs/architecture/remote-mcp-transport-reliability.md b/docs/architecture/remote-mcp-transport-reliability.md index 1ee8823..1e6fb83 100644 --- a/docs/architecture/remote-mcp-transport-reliability.md +++ b/docs/architecture/remote-mcp-transport-reliability.md @@ -37,9 +37,9 @@ The default reliability probe does not send a session termination request. after the probe completes and only when initialization returned a valid `MCP-Session-Id`. -`--require-remote-reliability` is a result gate, not network consent. It is -invalid unless remote runtime probing is enabled. The command fails unless the -remote reliability scorecard reaches `pass`. +`--require-remote-reliability` is a result gate, not network consent, and +requires `--runtime --allow-network`. It fails unless every attempted remote +reliability scorecard passes. Local-only runs are unaffected. ## Probe Sequence @@ -57,9 +57,9 @@ remote reliability scorecard reaches `pass`. using `Last-Event-ID`. Honor a valid server-provided SSE `retry` delay within the remaining probe deadline. Do not reconnect when no event identifier is observed. -8. If a subsequent request returns HTTP 404 for an issued session identifier, - discard that identifier and make one bounded re-initialization attempt - without it. Never restart a session more than once. +8. If a GET carrying an issued session identifier returns HTTP 404, discard + that identifier and make one bounded re-initialization attempt without it. + Never restart a session more than once. 9. If lifecycle consent is enabled and a current session exists, send one bounded `DELETE` request with the protocol and session headers. @@ -89,11 +89,12 @@ rules. ### Expired Session -HTTP 404 on a request carrying an issued session identifier means the session +HTTP 404 from a GET carrying an issued session identifier means the session expired or was terminated. The probe discards the stale identifier and performs one fresh initialize request without a session header. A successful restart continues with the new session; a second session-expiry response fails restart -reliability. +reliability. A DELETE 404 never restarts or retries and is a termination +failure. ### DELETE diff --git a/docs/guides/github-action.md b/docs/guides/github-action.md index 5d7e173..3ad4614 100644 --- a/docs/guides/github-action.md +++ b/docs/guides/github-action.md @@ -17,10 +17,10 @@ Remote MCP checks are off by default. Set `runtime: "true"` and give explicit ne runtime: "true" allow-network: "true" allow-local-network: "true" # Remove for public endpoints. - require-remote-reliability: "true" # Fails unless the bounded reliability scorecard passes. + require-remote-reliability: "true" # Fails unless every attempted reliability scorecard passes. ``` -The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. `require-remote-reliability` is a strict result gate, not network consent. Keep `allow-session-lifecycle: "false"` (the default) unless the workflow explicitly authorizes one bounded, state-changing session `DELETE` after a valid session is issued. Remote probes redact diagnostics and retain no raw remote content; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](../architecture/remote-mcp-transport-reliability.md) for SSRF, OAuth metadata-discovery, SSE, and lifecycle boundaries. +The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. `require-remote-reliability` is a strict result gate, not network consent: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. Keep `allow-session-lifecycle: "false"` (the default) unless the workflow explicitly authorizes one bounded, state-changing session `DELETE` after a valid session is issued. Remote probes redact diagnostics and retain no raw remote content; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](../architecture/remote-mcp-transport-reliability.md) for SSRF, OAuth metadata-discovery, SSE, and lifecycle boundaries. ## Recommended Workflow diff --git a/docs/guides/release-gating.md b/docs/guides/release-gating.md index 1ba51cc..83d703c 100644 --- a/docs/guides/release-gating.md +++ b/docs/guides/release-gating.md @@ -66,7 +66,7 @@ Remote MCP probing remains opt-in: it requires `--runtime --allow-network`, plus node dist/cli.js check ./path/to/plugin --runtime --allow-network --require-remote-reliability ``` -`--require-remote-reliability` is a strict result gate and grants no network consent; the command passes only when the reliability scorecard passes. Keep `--allow-session-lifecycle` off for ordinary release gates. That opt-in is state-changing and permits one bounded session `DELETE` only when initialization supplied a valid session identifier. +`--require-remote-reliability` is a strict result gate and grants no network consent; it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. Keep `--allow-session-lifecycle` off for ordinary release gates. That opt-in is state-changing and permits one bounded session `DELETE` only when initialization supplied a valid session identifier. Docker mode currently supports Node.js stdio MCP servers. It uses a read-only package mount and container filesystem, no network, an unprivileged user, dropped capabilities, bounded resources, and a limited writable `/tmp`. It fails closed and does not fall back to native execution. diff --git a/docs/security/runtime-approval-and-sandboxing.md b/docs/security/runtime-approval-and-sandboxing.md index dda7fab..ea4fb9c 100644 --- a/docs/security/runtime-approval-and-sandboxing.md +++ b/docs/security/runtime-approval-and-sandboxing.md @@ -10,7 +10,7 @@ Native probing preserves the existing host execution behavior: codex-plugin-doctor check ./plugin --runtime ``` -Remote MCP probing additionally requires `--allow-network`; loopback endpoints require `--allow-local-network`. Its transport reliability check is bounded and does not retain raw session IDs, event IDs, retry values, SSE data, or response bodies. `--allow-session-lifecycle` is false by default and is state-changing: it allows one bounded session `DELETE` only after a valid session identifier is issued. `--require-remote-reliability` is a strict pass gate, not network consent. +Remote MCP probing additionally requires `--allow-network`; loopback endpoints require `--allow-local-network`. Its transport reliability check is bounded and does not retain raw session IDs, event IDs, retry values, SSE data, or response bodies. `--allow-session-lifecycle` is false by default and is state-changing: it allows one bounded session `DELETE` only after a valid session identifier is issued. `--require-remote-reliability` is a strict pass gate, not network consent: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. Docker probing is explicit and currently supports local Node.js stdio servers: diff --git a/tests/public-readiness.test.ts b/tests/public-readiness.test.ts index 403aa7b..d2e4c51 100644 --- a/tests/public-readiness.test.ts +++ b/tests/public-readiness.test.ts @@ -128,6 +128,12 @@ describe("public repository readiness", () => { expect(reliability).toContain("--runtime --allow-network"); expect(reliability).toContain("one bounded `DELETE` request"); expect(reliability).toContain("does not claim delivery guarantees"); + expect(reliability).toContain("GET carrying an issued session identifier"); + expect(reliability).toContain("DELETE 404 never restarts or retries"); + for (const document of [reliability, readme, readiness, releaseGating, runtimeSecurity, actionGuide]) { + expect(document).toMatch(/every attempted remote\s+reliability scorecard passes/); + expect(document).toContain("Local-only runs are unaffected."); + } expect(releaseGating).toContain("--require-remote-reliability"); expect(runtimeSecurity).toContain("--allow-session-lifecycle"); expect(security).toContain("runner or host egress controls"); diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index fc30347..ee4b759 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { BoundedHttpError, type BoundedHttpRequestOptions, type BoundedHttpResponse } from "../src/core/bounded-http-client.js"; import { probeRemoteTransportReliability } from "../src/core/remote-transport-reliability.js"; +import { renderJsonReport } from "../src/reporting/render-json-report.js"; import { buildMarkdownReport } from "../src/reporting/render-markdown-report.js"; import { renderTextReport } from "../src/reporting/render-text-report.js"; @@ -101,7 +102,7 @@ describe("probeRemoteTransportReliability", () => { }; for (const output of [ - JSON.stringify(report), + renderJsonReport(report, { runtimeProbeEnabled: true }), renderTextReport(report), buildMarkdownReport(report, { runtimeProbeEnabled: true }) ]) { From b0dd4a3d9019b171b26210887f62fe894c0778d9 Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 16:47:29 +0300 Subject: [PATCH 14/15] chore: prepare v1.53.0 release --- CHANGELOG.md | 17 ++++++++++++++ README.md | 4 ++-- docs/guides/github-action.md | 44 ++++++++++++++++++------------------ package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 44 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13aa4f3..e6cf235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to `codex-plugin-doctor` are documented here. This changelog groups the shipped work into product-level release blocks instead of repeating every low-level git diff in isolation. +## [1.53.0] - 2026-07-26 + +### Added + +- added a bounded remote MCP transport-reliability readiness check that performs a post-initialize SSE `GET`, preserves session propagation, and observes bounded resume and session-restart paths +- added `allow-session-lifecycle` and `require-remote-reliability` GitHub Action inputs for explicit session-lifecycle approval and strict reliability-result gating + +### Changed + +- extended remote runtime reports, scorecards, output contracts, and release evidence with additive transport-reliability results +- made `--require-remote-reliability` fail when an attempted remote reliability scorecard does not pass; the flag grants no network consent and does not affect local-only runs + +### Security + +- kept remote session termination opt-in: after a valid session identifier, `--allow-session-lifecycle` permits at most one bounded session `DELETE` +- hardened remote reliability diagnostics to redact response content and sensitive transport details + ## [1.52.0] - 2026-07-25 ### Added diff --git a/README.md b/README.md index 41bf9a0..11bb8c4 100644 --- a/README.md +++ b/README.md @@ -442,9 +442,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: Esquetta/CodexPluginDoctor@v1.52.0 + - uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" policy: codex-publish diff --git a/docs/guides/github-action.md b/docs/guides/github-action.md index 3ad4614..0a20578 100644 --- a/docs/guides/github-action.md +++ b/docs/guides/github-action.md @@ -38,9 +38,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: Esquetta/CodexPluginDoctor@v1.52.0 + - uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" policy: codex-publish @@ -67,9 +67,9 @@ Every action run also writes `codex-plugin-doctor-action-manifest.json`. The man Use SARIF when repository security tooling should ingest validation findings. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . sarif: "true" ``` @@ -81,9 +81,9 @@ The action writes `codex-plugin-doctor.sarif` into `output-dir`. Uploading it to Use artifact and summary controls when the workflow needs custom retention or wants to disable generated report uploads. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . output-dir: doctor-ci-reports artifact-name: codex-plugin-doctor-reports @@ -118,11 +118,11 @@ The action also exposes these workflow outputs for follow-up steps: Use review bundle artifacts when a pull request or release workflow should preserve signed runtime approval, runtime policy, attestation, and release evidence handoff files. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 env: CODEX_PLUGIN_DOCTOR_SIGNING_KEY: ${{ secrets.CODEX_PLUGIN_DOCTOR_SIGNING_KEY }} with: - version: "1.52.0" + version: "1.53.0" path: . review-bundle: "true" review-bundle-verify: "true" @@ -153,9 +153,9 @@ The CLI can produce badge output for release notes, README automation, or a stat Use a private corpus metrics manifest to measure reviewed precision, recall, and false-positive share in CI. The action writes only the public-safe metrics report into its artifact directory; snapshots, manifest contents, local paths, and review notes are not copied. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . corpus-metrics-manifest: ../private-corpus/metrics.json ``` @@ -163,9 +163,9 @@ Use a private corpus metrics manifest to measure reviewed precision, recall, and This writes `corpus-metrics.json`. To compare the result with a retained report and fail the job on regression: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . corpus-metrics-manifest: ../private-corpus/metrics.json corpus-metrics-baseline: .doctor-baselines/corpus-metrics.json @@ -194,9 +194,9 @@ The history file is newline-delimited JSON. Store it as an artifact, cache, or r The composite action can also append history directly: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" history: validation-history.jsonl @@ -216,9 +216,9 @@ Use profiles when a consuming workflow needs a named validation policy instead o The composite action can pass profiles directly: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . profile: publish ``` @@ -228,9 +228,9 @@ The composite action can pass profiles directly: Use policy presets when a workflow should apply one of the opinionated release gates without adding a local `.codex-doctor.json`. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . policy: codex-publish ``` @@ -242,9 +242,9 @@ Supported policy values are `codex-publish`, `mcp-strict`, and `security`. The C Use installed-cache mode only in environments where Codex plugins are already available on the runner. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" installed: "true" filter: github runtime: "false" @@ -255,9 +255,9 @@ Use installed-cache mode only in environments where Codex plugins are already av Pin both the action ref and npm package version for reproducible CI: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" ``` Use `version: "latest"` only when the consuming repository intentionally wants automatic CLI upgrades. diff --git a/package-lock.json b/package-lock.json index 68e1f2a..7572338 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "license": "MIT", "bin": { "codex-plugin-doctor": "dist/cli.js" diff --git a/package.json b/package.json index a6f296c..6f67f53 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "description": "CLI-first validator for Codex plugins, skills, and MCP package surfaces with runtime MCP protocol validation.", "type": "module", "main": "./dist/index.js", From 04790e36ac665f0db33f421335c58a7121803a3d Mon Sep 17 00:00:00 2001 From: Furkan Date: Sun, 26 Jul 2026 17:11:54 +0300 Subject: [PATCH 15/15] fix: enforce remote MCP session restart invariants --- src/core/remote-mcp-probe.ts | 8 +- src/core/remote-transport-reliability.ts | 49 +++++++------ tests/mcp-command.test.ts | 2 +- tests/remote-mcp-probe.test.ts | 85 ++++++++++++++++++++++ tests/remote-transport-reliability.test.ts | 27 +++++++ 5 files changed, 147 insertions(+), 24 deletions(-) diff --git a/src/core/remote-mcp-probe.ts b/src/core/remote-mcp-probe.ts index af39972..725582f 100644 --- a/src/core/remote-mcp-probe.ts +++ b/src/core/remote-mcp-probe.ts @@ -140,6 +140,10 @@ function isValidInitializeResponse(message: JsonObject): boolean { ); } +function isInitializedNotificationAcknowledged(response: BoundedHttpResponse): boolean { + return response.statusCode === 202 && response.body.length === 0; +} + function validSessionId(value: string | null): boolean { return value !== null && /^[\x21-\x7e]+$/.test(value); } @@ -312,7 +316,7 @@ export async function probeRemoteMcpServer( ...(sessionId === null ? {} : { "MCP-Session-Id": sessionId }) } }); - if (initializedResponse.statusCode < 200 || initializedResponse.statusCode >= 300) { + if (!isInitializedNotificationAcknowledged(initializedResponse)) { scorecard.protocolHeaders = "fail"; findings.push(failure( "plugin.runtime.remote.initialized.failed", @@ -380,7 +384,7 @@ export async function probeRemoteMcpServer( ...(replacementSessionId === null ? {} : { "MCP-Session-Id": replacementSessionId }) } }); - if (restartInitialized.statusCode < 200 || restartInitialized.statusCode >= 300) { + if (!isInitializedNotificationAcknowledged(restartInitialized)) { throw new Error("restart initialized failed"); } return replacementSessionId; diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts index 3aca5d8..596730e 100644 --- a/src/core/remote-transport-reliability.ts +++ b/src/core/remote-transport-reliability.ts @@ -33,6 +33,7 @@ interface RequestResult { response?: BoundedHttpResponse; error?: unknown; restartFailed?: boolean; + sessionRestarted?: boolean; } function createScorecard(): RemoteTransportReliabilityScorecard { @@ -153,13 +154,14 @@ export async function probeRemoteTransportReliability( extraHeaders: Record = {}, stopAfter?: (body: Buffer) => boolean ): Promise => { + let sessionRestarted = false; for (let attempt = 0; attempt < 2; attempt += 1) { const sessionId = currentSessionId; const headers = { Accept: "text/event-stream", "MCP-Protocol-Version": options.protocolVersion, ...(sessionId === null ? {} : { "MCP-Session-Id": sessionId }), - ...extraHeaders + ...(sessionRestarted ? {} : extraHeaders) }; if (sessionId !== null) { scorecard.sessionPropagation = "pass"; @@ -181,6 +183,7 @@ export async function probeRemoteTransportReliability( return { restartFailed: true }; } restarted = true; + sessionRestarted = true; currentSessionId = null; try { const replacementSessionId = await options.reinitialize(requestWithinBudget); @@ -195,7 +198,7 @@ export async function probeRemoteTransportReliability( } continue; } - return { response }; + return { response, sessionRestarted }; } return { restartFailed: true }; }; @@ -310,28 +313,32 @@ export async function probeRemoteTransportReliability( scorecard.resumability = "fail"; return finalize(scorecard, findings); } - const resumeObservation = classifySseResponse(resumed.response, "resume", findings); - if (resumeObservation === null || resumeObservation.malformed) { - scorecard.resumability = "fail"; - if (resumeObservation?.malformed) { + if (resumed.sessionRestarted) { + scorecard.resumability = "skipped"; + } else { + const resumeObservation = classifySseResponse(resumed.response, "resume", findings); + if (resumeObservation === null || resumeObservation.malformed) { + scorecard.resumability = "fail"; + if (resumeObservation?.malformed) { + findings.push(finding( + "plugin.runtime.remote.reliability.resume.malformed", + "fail", + "The remote MCP endpoint emitted malformed SSE framing after a resume request.", + "Return complete SSE events with valid id and retry fields after reconnecting." + )); + } + return finalize(scorecard, findings); + } + scorecard.resumability = resumeObservation.complete ? "pass" : "warn"; + if (!resumeObservation.complete) { findings.push(finding( - "plugin.runtime.remote.reliability.resume.malformed", - "fail", - "The remote MCP endpoint emitted malformed SSE framing after a resume request.", - "Return complete SSE events with valid id and retry fields after reconnecting." + "plugin.runtime.remote.reliability.resume.inconclusive", + "warn", + "The remote MCP SSE resume response ended before a complete event could be observed.", + "Emit complete SSE events after accepting a reconnect." )); + return finalize(scorecard, findings); } - return finalize(scorecard, findings); - } - scorecard.resumability = resumeObservation.complete ? "pass" : "warn"; - if (!resumeObservation.complete) { - findings.push(finding( - "plugin.runtime.remote.reliability.resume.inconclusive", - "warn", - "The remote MCP SSE resume response ended before a complete event could be observed.", - "Emit complete SSE event frames after accepting a reconnect." - )); - return finalize(scorecard, findings); } } } diff --git a/tests/mcp-command.test.ts b/tests/mcp-command.test.ts index 33a0c5d..7bd8034 100644 --- a/tests/mcp-command.test.ts +++ b/tests/mcp-command.test.ts @@ -90,7 +90,7 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean; sess return; } - response.writeHead(204); + response.writeHead(202); response.end(); }); }); diff --git a/tests/remote-mcp-probe.test.ts b/tests/remote-mcp-probe.test.ts index ee4e7f7..bcbe809 100644 --- a/tests/remote-mcp-probe.test.ts +++ b/tests/remote-mcp-probe.test.ts @@ -146,6 +146,45 @@ describe("probeRemoteMcpServer", () => { assertPrivate(result); }); + it.each([ + ["HTTP 200", 200, ""], + ["another 2xx status", 204, ""], + ["a non-empty HTTP 202 response", 202, "notification-body-secret-sentinel"] + ])("rejects %s for the initial initialized notification", async (_name, statusCode, responseBody) => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(statusCode); + response.end(responseBody); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.initialized.failed", severity: "fail" }) + ]); + expect(result.scorecard).toMatchObject({ protocolHeaders: "fail", overall: "fail" }); + expect(JSON.stringify(result)).not.toContain("notification-body-secret-sentinel"); + assertPrivate(result); + }); + it("merges reliability scorecards across remote servers field by field", async () => { const port = await startServer((request, response) => { let body = ""; @@ -330,6 +369,52 @@ describe("probeRemoteMcpServer", () => { assertPrivate(result); }); + it.each([ + ["HTTP 200", 200, ""], + ["another 2xx status", 204, ""], + ["a non-empty HTTP 202 response", 202, "notification-body-secret-sentinel"] + ])("rejects %s for the replacement session initialized notification", async (_name, statusCode, responseBody) => { + let initializeCount = 0; + const requests: Array<{ method: string; body: string }> = []; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", body }); + if (request.method === "GET") { + response.writeHead(404); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + initializeCount += 1; + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": initializeCount === 1 + ? "session-secret-sentinel" + : "replacement-session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(initializeCount === 1 ? 202 : statusCode); + response.end(initializeCount === 1 ? "" : responseBody); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" }) + ]); + expect(result.scorecard.reliability).toMatchObject({ sessionRestart: "fail", overall: "fail" }); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET", "POST", "POST"]); + expect(JSON.stringify(result)).not.toContain("notification-body-secret-sentinel"); + assertPrivate(result); + }); + it("skips SSE primer events until the initialize response without waiting for the stream to close", async () => { const port = await startServer((request, response) => { let body = ""; diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts index ee4b759..3d728e8 100644 --- a/tests/remote-transport-reliability.test.ts +++ b/tests/remote-transport-reliability.test.ts @@ -249,6 +249,33 @@ describe("probeRemoteTransportReliability", () => { assertRedacted(result); }); + it("does not carry a resume cursor into a replacement MCP session", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\n\n"), + response(404), + response(405) + ]); + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => "replacement-session-secret-sentinel" + }); + + expect(fixture.requests).toHaveLength(3); + expect(fixture.requests[1]?.headers?.["Last-Event-ID"]).toBe("event-secret-sentinel"); + expect(fixture.requests[2]?.headers).toMatchObject({ + "MCP-Session-Id": "replacement-session-secret-sentinel" + }); + expect(fixture.requests[2]?.headers?.["Last-Event-ID"]).toBeUndefined(); + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ + sessionRestart: "pass", + resumability: "skipped", + overall: "pass" + }); + assertRedacted(result); + }); + it("counts both restart POST requests against the shared reliability request budget", async () => { const fixture = scriptedRequest([ response(404),