diff --git a/CHANGELOG.md b/CHANGELOG.md index 13aa4f3..e6cf235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to `codex-plugin-doctor` are documented here. This changelog groups the shipped work into product-level release blocks instead of repeating every low-level git diff in isolation. +## [1.53.0] - 2026-07-26 + +### Added + +- added a bounded remote MCP transport-reliability readiness check that performs a post-initialize SSE `GET`, preserves session propagation, and observes bounded resume and session-restart paths +- added `allow-session-lifecycle` and `require-remote-reliability` GitHub Action inputs for explicit session-lifecycle approval and strict reliability-result gating + +### Changed + +- extended remote runtime reports, scorecards, output contracts, and release evidence with additive transport-reliability results +- made `--require-remote-reliability` fail when an attempted remote reliability scorecard does not pass; the flag grants no network consent and does not affect local-only runs + +### Security + +- kept remote session termination opt-in: after a valid session identifier, `--allow-session-lifecycle` permits at most one bounded session `DELETE` +- hardened remote reliability diagnostics to redact response content and sensitive transport details + ## [1.52.0] - 2026-07-25 ### Added diff --git a/README.md b/README.md index 2687291..11bb8c4 100644 --- a/README.md +++ b/README.md @@ -83,14 +83,18 @@ Runtime MCP validation with `--runtime`: ### Remote MCP Readiness -Remote MCP runtime probing is disabled until you explicitly pass `--allow-network`. Local endpoints also require `--allow-local-network`: +Remote MCP runtime probing is disabled until you explicitly pass `--runtime --allow-network`. Loopback endpoints also require `--allow-local-network`: ```bash codex-plugin-doctor check ./remote-mcp --runtime --allow-network codex-plugin-doctor check ./remote-mcp --runtime --allow-network --allow-local-network ``` -The probe makes only bounded, read-only protocol and OAuth metadata-discovery requests, redacts report output, and applies SSRF controls. It does not authenticate or follow redirects. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md). +The probe makes only bounded protocol and OAuth metadata-discovery requests, redacts report output, and applies SSRF controls. It does not authenticate or follow redirects. + +Remote transport reliability adds one bounded SSE GET after initialization. HTTP `200` must be `text/event-stream`; HTTP `405` is compliant when the endpoint does not offer server-to-client SSE. At most one SSE resume and one session restart are attempted, and no remote response content is retained. This is a bounded readiness check, not a live interoperability, delivery-guarantee, or load-test claim. + +`--allow-session-lifecycle` is disabled by default and is state-changing: only after a valid `MCP-Session-Id` it permits one bounded session `DELETE`. `--require-remote-reliability` is a strict result gate: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. It grants no network consent, so `--runtime --allow-network` (and loopback consent when applicable) remain required. See [Remote MCP Readiness](./docs/architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](./docs/architecture/remote-mcp-transport-reliability.md). Output formats: @@ -438,9 +442,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: Esquetta/CodexPluginDoctor@v1.52.0 + - uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" policy: codex-publish diff --git a/action.yml b/action.yml index 38eca5c..4e6d647 100644 --- a/action.yml +++ b/action.yml @@ -22,6 +22,14 @@ inputs: description: Explicitly allow remote MCP runtime probes to contact loopback endpoints only (localhost, 127.0.0.0/8, or ::1). Private, link-local, multicast, unspecified, reserved, and NAT64 ranges remain blocked. required: false default: "false" + allow-session-lifecycle: + description: Explicitly allow remote MCP runtime probing to send one state-changing session termination request when a valid session is issued. + required: false + default: "false" + require-remote-reliability: + description: Fail the validation result unless every attempted remote MCP reliability scorecard passes; this does not grant network access. + required: false + default: "false" installed: description: Validate plugins from the local Codex plugin cache. required: false @@ -170,6 +178,8 @@ runs: env: ALLOW_NETWORK_INPUT: ${{ inputs['allow-network'] }} ALLOW_LOCAL_NETWORK_INPUT: ${{ inputs['allow-local-network'] }} + ALLOW_SESSION_LIFECYCLE_INPUT: ${{ inputs['allow-session-lifecycle'] }} + REQUIRE_REMOTE_RELIABILITY_INPUT: ${{ inputs['require-remote-reliability'] }} CORPUS_METRICS_MANIFEST_INPUT: ${{ inputs['corpus-metrics-manifest'] }} CORPUS_METRICS_BASELINE_INPUT: ${{ inputs['corpus-metrics-baseline'] }} CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT: ${{ inputs['corpus-metrics-fail-on-regression'] }} @@ -217,6 +227,14 @@ runs: args+=(--allow-local-network) fi + if [[ "$ALLOW_SESSION_LIFECYCLE_INPUT" == "true" ]]; then + args+=(--allow-session-lifecycle) + fi + + if [[ "$REQUIRE_REMOTE_RELIABILITY_INPUT" == "true" ]]; then + args+=(--require-remote-reliability) + fi + if [[ -n "${{ inputs.config }}" ]]; then args+=(--config "${{ inputs.config }}") fi diff --git a/docs/README.md b/docs/README.md index c1edf02..81704a9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ This directory contains public documentation for users, contributors, and securi - [Runtime Sandbox and External Corpus](architecture/runtime-sandbox-and-external-corpus.md) - [MCP 2025-11 Conformance](architecture/mcp-2025-11-conformance.md) - [Remote MCP Readiness](architecture/remote-mcp-readiness.md) +- [Remote MCP Transport Reliability](architecture/remote-mcp-transport-reliability.md) - [Real-World Corpus Quality Metrics](architecture/real-world-corpus-quality-metrics.md) - [Corpus Metrics Regression Diff](architecture/corpus-metrics-regression-diff.md) diff --git a/docs/architecture/remote-mcp-readiness.md b/docs/architecture/remote-mcp-readiness.md index d6ec781..f7d199e 100644 --- a/docs/architecture/remote-mcp-readiness.md +++ b/docs/architecture/remote-mcp-readiness.md @@ -17,7 +17,13 @@ The same consent is available in the GitHub Action through `allow-network: "true ## Read-Only Scope -The probe uses a bounded HTTP request for MCP initialization and only follows the OAuth metadata-discovery path advertised by an unauthenticated challenge. It never sends credentials or tokens, and reporting redacts sensitive values, response bodies, session identifiers, and authorization metadata. +The probe uses bounded HTTP requests for MCP initialization and only follows the OAuth metadata-discovery path advertised by an unauthenticated challenge. It never sends credentials or tokens, and reporting redacts sensitive values, response bodies, session identifiers, SSE event identifiers, SSE retry values, and authorization metadata. + +## Transport Reliability + +The optional reliability scorecard runs only within the same `--runtime --allow-network` consent boundary; loopback endpoints still need `--allow-local-network`. It makes one bounded GET request after initialization. A `200` response must use `text/event-stream`; a `405` response is compliant when server-to-client SSE is unsupported. The probe can make at most one SSE resume and one session restart, and it retains no raw response body or SSE data. It is a bounded readiness check, not a claim of live remote interoperability, replay correctness, delivery guarantees, or load capacity. + +`--allow-session-lifecycle` is false by default. It is state-changing and permits one bounded `DELETE` only after initialization supplies a valid `MCP-Session-Id`. `--require-remote-reliability` is a strict result gate: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. The flag grants no network or loopback consent. See [Remote MCP Transport Reliability](remote-mcp-transport-reliability.md) for the protocol sequence and classifications. ## SSRF Controls @@ -30,7 +36,6 @@ These checks reduce SSRF exposure but cannot account for every network topology. - authenticated OAuth - custom headers - remote tool/resource/prompt/task calls -- GET SSE/resumability - redirects Use a dedicated MCP client with its own authorization and network policy when any of these capabilities are required. diff --git a/docs/architecture/remote-mcp-transport-reliability.md b/docs/architecture/remote-mcp-transport-reliability.md new file mode 100644 index 0000000..1e6fb83 --- /dev/null +++ b/docs/architecture/remote-mcp-transport-reliability.md @@ -0,0 +1,212 @@ +# Remote MCP Transport Reliability + +## Purpose + +Codex Plugin Doctor verifies whether an explicitly approved remote MCP endpoint +behaves safely and predictably across Streamable HTTP session and SSE lifecycle +boundaries. The probe is a bounded readiness check, not a general-purpose MCP +client or load test. + +## Goals + +- validate protocol-compliant GET behavior after initialization +- validate session propagation without exposing session identifiers +- observe bounded SSE event framing and resumability when evidence is available +- classify disconnects, timeouts, retry delays, and one session restart without + retaining remote response content +- optionally terminate an initialized session after explicit user consent +- expose a stable, additive reliability scorecard and CI gate + +## Non-Goals + +- authenticated OAuth requests +- tool, resource, prompt, or task execution +- arbitrary custom request headers +- unbounded SSE subscriptions +- automatic retries beyond one evidence-driven resume attempt or one + protocol-required session restart +- performance, availability, or load testing + +## Consent Model + +Remote probing continues to require `--runtime --allow-network`. Loopback +endpoints additionally require `--allow-local-network`. + +The default reliability probe does not send a session termination request. +`--allow-session-lifecycle` explicitly permits one bounded `DELETE` request +after the probe completes and only when initialization returned a valid +`MCP-Session-Id`. + +`--require-remote-reliability` is a result gate, not network consent, and +requires `--runtime --allow-network`. It fails unless every attempted remote +reliability scorecard passes. Local-only runs are unaffected. + +## Probe Sequence + +1. Perform the existing bounded `initialize` request. +2. Validate `MCP-Session-Id` as visible ASCII when present. +3. Send `notifications/initialized` with the negotiated protocol and session + headers. +4. Send one bounded GET request with `Accept: text/event-stream`, the negotiated + protocol version, and the session identifier when present. +5. Accept HTTP 405 as a protocol-compliant declaration that the endpoint does + not offer a server-to-client SSE stream. +6. For HTTP 200, require `text/event-stream` and inspect only enough bytes to + identify a complete SSE event or reach the configured bounds. +7. If a valid SSE `id` field is observed, make one bounded reconnect attempt + using `Last-Event-ID`. Honor a valid server-provided SSE `retry` delay within + the remaining probe deadline. Do not reconnect when no event identifier is + observed. +8. If a GET carrying an issued session identifier returns HTTP 404, discard + that identifier and make one bounded re-initialization attempt without it. + Never restart a session more than once. +9. If lifecycle consent is enabled and a current session exists, send one bounded + `DELETE` request with the protocol and session headers. + +No step follows redirects. Every request reuses the existing DNS resolution, +peer matching, response-size, encoding, and timeout controls. +The complete probe has a fixed request-count ceiling; reconnect and session +restart paths cannot recurse. + +## HTTP Classification + +### GET + +- `200` with `text/event-stream`: supported; continue bounded SSE inspection +- `405`: compliant but unsupported; no resumability attempt +- any other status: fail +- `200` with another media type: fail + +An SSE connection that produces no complete event before the observation +deadline is inconclusive, not proof of malformed framing. A malformed complete +event, oversized response, unsafe peer change, or transport error fails the +relevant capability. + +A valid SSE `retry` field is respected before reconnecting. If its delay exceeds +the remaining bounded probe deadline, resumability is inconclusive and no late +request is sent. Invalid retry fields are ignored according to SSE parsing +rules. + +### Expired Session + +HTTP 404 from a GET carrying an issued session identifier means the session +expired or was terminated. The probe discards the stale identifier and performs +one fresh initialize request without a session header. A successful restart +continues with the new session; a second session-expiry response fails restart +reliability. A DELETE 404 never restarts or retries and is a termination +failure. + +### DELETE + +- any `2xx`: termination passed +- `405`: compliant but unsupported +- any other status or transport failure: termination failed + +DELETE classification is `skipped` when lifecycle consent is absent or no +session identifier was issued. + +## Resumability + +The probe attempts resumability only when the first bounded GET produces a +complete event with a non-empty valid `id` field. The exact event identifier is +held in memory only for the duration of the reconnect request and is sent as +`Last-Event-ID`. + +A reconnect confirms request propagation and bounded transport behavior. It +does not claim delivery guarantees or replay correctness because the probe does +not retain or compare application payloads. + +## Scorecard + +The existing remote runtime scorecard gains an additive `reliability` object: + +```json +{ + "getSse": "pass", + "sessionPropagation": "pass", + "resumability": "skipped", + "disconnectSafety": "pass", + "sessionRestart": "skipped", + "termination": "skipped", + "overall": "pass" +} +``` + +Capability fields use the existing `pass`, `warn`, `fail`, and `skipped` +statuses. `overall` is: + +- `fail` when any attempted reliability check fails +- `warn` when no check fails but an attempted check is inconclusive +- `pass` when all applicable checks are compliant +- `skipped` when reliability probing does not run + +Protocol-compliant unsupported behavior, such as GET or DELETE returning 405, +does not lower an otherwise passing score. + +## Data Handling + +The following values must never appear in text, Markdown, JSON, evidence, +transcript, error, or debug output: + +- raw `MCP-Session-Id` values +- raw `Last-Event-ID` values +- raw SSE `retry` values +- SSE event data +- remote response bodies +- authorization metadata beyond existing redacted readiness classifications + +Reports expose only classifications and stable finding identifiers. The probe +does not write remote values to disk. + +## HTTP Client Changes + +The bounded HTTP client may add only the capabilities required by this design: + +- allow the `Last-Event-ID` request header +- preserve safe status and response-header metadata on a timeout after response + headers arrive +- keep existing three-second and one-megabyte upper bounds + +It must not add redirects, cookies, authorization headers, connection pooling, +or configurable unsafe headers. + +## CLI And GitHub Action + +The `check`, release-check, release-evidence, and GitHub Action paths that +already expose remote runtime probing receive consistent inputs: + +- `--allow-session-lifecycle` +- `--require-remote-reliability` +- `allow-session-lifecycle` +- `require-remote-reliability` + +Help output and public documentation must state that lifecycle consent can +change remote server state. Existing commands remain unchanged when both new +flags are absent. + +## Findings + +New findings use the `plugin.runtime.remote.reliability.*` namespace and must +provide actionable remediation without including remote values. At minimum, +tests cover invalid GET status, invalid SSE media type, malformed event framing, +resume failure, and termination failure. + +## Verification + +The implementation requires deterministic local HTTP fixtures for: + +- GET returning 405 +- valid SSE with no event identifier +- valid SSE with an event identifier and one resume request +- valid SSE retry delay and an over-deadline retry delay +- invalid SSE content type and malformed framing +- session propagation on initialized, GET, resume, and DELETE requests +- session expiry followed by one successful or failed re-initialization +- timeout and disconnect behavior +- DELETE disabled, successful, unsupported, and failed states +- output redaction across text, Markdown, JSON, and release evidence +- CLI validation and GitHub Action input forwarding +- `--require-remote-reliability` pass, warn, fail, and skipped outcomes + +Completion requires focused tests, the full test suite, build, dependency audit, +package verification, and a clean consumer installation. diff --git a/docs/guides/github-action.md b/docs/guides/github-action.md index 5d052b2..0a20578 100644 --- a/docs/guides/github-action.md +++ b/docs/guides/github-action.md @@ -17,9 +17,10 @@ Remote MCP checks are off by default. Set `runtime: "true"` and give explicit ne runtime: "true" allow-network: "true" allow-local-network: "true" # Remove for public endpoints. + require-remote-reliability: "true" # Fails unless every attempted reliability scorecard passes. ``` -The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. Remote probes remain read-only and redact diagnostics; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) for SSRF and OAuth metadata-discovery boundaries. +The Action transfers these boolean inputs through environment-backed shell variables and a Bash argument array. `require-remote-reliability` is a strict result gate, not network consent: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. Keep `allow-session-lifecycle: "false"` (the default) unless the workflow explicitly authorizes one bounded, state-changing session `DELETE` after a valid session is issued. Remote probes redact diagnostics and retain no raw remote content; see [Remote MCP Readiness](../architecture/remote-mcp-readiness.md) and [Remote MCP Transport Reliability](../architecture/remote-mcp-transport-reliability.md) for SSRF, OAuth metadata-discovery, SSE, and lifecycle boundaries. ## Recommended Workflow @@ -37,9 +38,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: Esquetta/CodexPluginDoctor@v1.52.0 + - uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" policy: codex-publish @@ -66,9 +67,9 @@ Every action run also writes `codex-plugin-doctor-action-manifest.json`. The man Use SARIF when repository security tooling should ingest validation findings. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . sarif: "true" ``` @@ -80,9 +81,9 @@ The action writes `codex-plugin-doctor.sarif` into `output-dir`. Uploading it to Use artifact and summary controls when the workflow needs custom retention or wants to disable generated report uploads. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . output-dir: doctor-ci-reports artifact-name: codex-plugin-doctor-reports @@ -117,11 +118,11 @@ The action also exposes these workflow outputs for follow-up steps: Use review bundle artifacts when a pull request or release workflow should preserve signed runtime approval, runtime policy, attestation, and release evidence handoff files. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 env: CODEX_PLUGIN_DOCTOR_SIGNING_KEY: ${{ secrets.CODEX_PLUGIN_DOCTOR_SIGNING_KEY }} with: - version: "1.52.0" + version: "1.53.0" path: . review-bundle: "true" review-bundle-verify: "true" @@ -152,9 +153,9 @@ The CLI can produce badge output for release notes, README automation, or a stat Use a private corpus metrics manifest to measure reviewed precision, recall, and false-positive share in CI. The action writes only the public-safe metrics report into its artifact directory; snapshots, manifest contents, local paths, and review notes are not copied. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . corpus-metrics-manifest: ../private-corpus/metrics.json ``` @@ -162,9 +163,9 @@ Use a private corpus metrics manifest to measure reviewed precision, recall, and This writes `corpus-metrics.json`. To compare the result with a retained report and fail the job on regression: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . corpus-metrics-manifest: ../private-corpus/metrics.json corpus-metrics-baseline: .doctor-baselines/corpus-metrics.json @@ -193,9 +194,9 @@ The history file is newline-delimited JSON. Store it as an artifact, cache, or r The composite action can also append history directly: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . runtime: "true" history: validation-history.jsonl @@ -215,9 +216,9 @@ Use profiles when a consuming workflow needs a named validation policy instead o The composite action can pass profiles directly: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . profile: publish ``` @@ -227,9 +228,9 @@ The composite action can pass profiles directly: Use policy presets when a workflow should apply one of the opinionated release gates without adding a local `.codex-doctor.json`. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" path: . policy: codex-publish ``` @@ -241,9 +242,9 @@ Supported policy values are `codex-publish`, `mcp-strict`, and `security`. The C Use installed-cache mode only in environments where Codex plugins are already available on the runner. ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" installed: "true" filter: github runtime: "false" @@ -254,9 +255,9 @@ Use installed-cache mode only in environments where Codex plugins are already av Pin both the action ref and npm package version for reproducible CI: ```yaml -- uses: Esquetta/CodexPluginDoctor@v1.52.0 +- uses: Esquetta/CodexPluginDoctor@v1.53.0 with: - version: "1.52.0" + version: "1.53.0" ``` Use `version: "latest"` only when the consuming repository intentionally wants automatic CLI upgrades. diff --git a/docs/guides/release-gating.md b/docs/guides/release-gating.md index 9fce94a..83d703c 100644 --- a/docs/guides/release-gating.md +++ b/docs/guides/release-gating.md @@ -58,6 +58,16 @@ node dist/cli.js check ./path/to/plugin --json --runtime --output codex-plugin-d node dist/cli.js check ./path/to/plugin --json --runtime --sandbox docker --output codex-plugin-doctor-runtime-report.json ``` +### Remote Transport Reliability + +Remote MCP probing remains opt-in: it requires `--runtime --allow-network`, plus `--allow-local-network` for loopback endpoints. The reliability probe uses bounded requests, accepts HTTP `405` as compliant when GET SSE is unsupported, and makes no more than one resume or session restart. It does not establish live interoperability, delivery, or load-test guarantees. + +```bash +node dist/cli.js check ./path/to/plugin --runtime --allow-network --require-remote-reliability +``` + +`--require-remote-reliability` is a strict result gate and grants no network consent; it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. Keep `--allow-session-lifecycle` off for ordinary release gates. That opt-in is state-changing and permits one bounded session `DELETE` only when initialization supplied a valid session identifier. + Docker mode currently supports Node.js stdio MCP servers. It uses a read-only package mount and container filesystem, no network, an unprivileged user, dropped capabilities, bounded resources, and a limited writable `/tmp`. It fails closed and does not fall back to native execution. ### Release Readiness diff --git a/docs/security/runtime-approval-and-sandboxing.md b/docs/security/runtime-approval-and-sandboxing.md index b484096..ea4fb9c 100644 --- a/docs/security/runtime-approval-and-sandboxing.md +++ b/docs/security/runtime-approval-and-sandboxing.md @@ -10,6 +10,8 @@ Native probing preserves the existing host execution behavior: codex-plugin-doctor check ./plugin --runtime ``` +Remote MCP probing additionally requires `--allow-network`; loopback endpoints require `--allow-local-network`. Its transport reliability check is bounded and does not retain raw session IDs, event IDs, retry values, SSE data, or response bodies. `--allow-session-lifecycle` is false by default and is state-changing: it allows one bounded session `DELETE` only after a valid session identifier is issued. `--require-remote-reliability` is a strict pass gate, not network consent: it fails unless every attempted remote reliability scorecard passes. Local-only runs are unaffected. + Docker probing is explicit and currently supports local Node.js stdio servers: ```bash diff --git a/package-lock.json b/package-lock.json index 68e1f2a..7572338 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "license": "MIT", "bin": { "codex-plugin-doctor": "dist/cli.js" diff --git a/package.json b/package.json index a6f296c..6f67f53 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codex-plugin-doctor", - "version": "1.52.0", + "version": "1.53.0", "description": "CLI-first validator for Codex plugins, skills, and MCP package surfaces with runtime MCP protocol validation.", "type": "module", "main": "./dist/index.js", diff --git a/src/core/bounded-http-client.ts b/src/core/bounded-http-client.ts index df6114b..5555b77 100644 --- a/src/core/bounded-http-client.ts +++ b/src/core/bounded-http-client.ts @@ -9,6 +9,7 @@ const DEFAULT_MAX_RESPONSE_BYTES = 1_024 * 1_024; const allowedRequestHeaders = new Set([ "accept", "content-type", + "last-event-id", "mcp-protocol-version", "mcp-session-id", "user-agent" @@ -199,7 +200,12 @@ export async function requestBoundedHttp( let response: http.IncomingMessage | undefined; let settled = false; const timeout = setTimeout(() => { - fail(new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote HTTP request timed out.")); + fail(new BoundedHttpError( + "REMOTE_HTTP_TIMEOUT", + "Remote HTTP request timed out.", + response?.statusCode, + response ? selectSafeHeaders(response.headers) : undefined + )); }, remainingMs); const fail = (error: Error): void => { diff --git a/src/core/doctor-export-bundle.ts b/src/core/doctor-export-bundle.ts index 15ec805..f415936 100644 --- a/src/core/doctor-export-bundle.ts +++ b/src/core/doctor-export-bundle.ts @@ -46,7 +46,9 @@ export function redactValue(value: unknown): unknown { return Object.fromEntries( Object.entries(value).map(([key, nestedValue]) => [ key, - redactValue(nestedValue) + /^(?:mcp[-_]?session[-_]?id|session[-_]?id|last[-_]?event[-_]?id|event[-_]?id|retry|(?:sse[-_]?)?data)$/i.test(key) + ? "[REDACTED_SECRET]" + : redactValue(nestedValue) ]) ); } diff --git a/src/core/output-contract.ts b/src/core/output-contract.ts index c3e7d7f..c5daeac 100644 --- a/src/core/output-contract.ts +++ b/src/core/output-contract.ts @@ -50,6 +50,11 @@ const runtimeCapabilityStatusSchema = { enum: ["pass", "fail", "warn", "skipped", "unsupported"] }; +const remoteTransportReliabilityStatusSchema = { + type: "string", + enum: ["pass", "warn", "fail", "skipped"] +}; + const runtimeConformanceSchema = { type: "object", properties: { @@ -81,6 +86,29 @@ const runtimeConformanceSchema = { additionalProperties: false }; +const remoteTransportReliabilitySchema = { + type: "object", + properties: { + getSse: remoteTransportReliabilityStatusSchema, + sessionPropagation: remoteTransportReliabilityStatusSchema, + resumability: remoteTransportReliabilityStatusSchema, + disconnectSafety: remoteTransportReliabilityStatusSchema, + sessionRestart: remoteTransportReliabilityStatusSchema, + termination: remoteTransportReliabilityStatusSchema, + overall: remoteTransportReliabilityStatusSchema + }, + required: [ + "getSse", + "sessionPropagation", + "resumability", + "disconnectSafety", + "sessionRestart", + "termination", + "overall" + ], + additionalProperties: false +}; + const remoteRuntimeScorecardSchema = { type: "object", properties: { @@ -94,6 +122,7 @@ const remoteRuntimeScorecardSchema = { }, protocolHeaders: runtimeCapabilityStatusSchema, authorization: runtimeCapabilityStatusSchema, + reliability: remoteTransportReliabilitySchema, overall: { type: "string", enum: ["pass", "warn", "fail", "skipped"] diff --git a/src/core/release-check.ts b/src/core/release-check.ts index 60fd61e..1f7b1e5 100644 --- a/src/core/release-check.ts +++ b/src/core/release-check.ts @@ -44,6 +44,8 @@ export interface BuildReleaseCheckOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeSandbox?: RuntimeSandboxMode; runCheck?: (targetPath: string, options: CheckOptions) => Promise; } @@ -177,6 +179,8 @@ export async function buildReleaseCheck( runtime: runtimeProbeEnabled, ...(options.allowNetwork ? { allowNetwork: true } : {}), ...(options.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(options.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(options.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(options.runtimeSandbox ? { runtimeSandbox: options.runtimeSandbox } : {}) }); const securityResult = await buildSecurityAudit(resolvedPath); diff --git a/src/core/release-evidence.ts b/src/core/release-evidence.ts index 71031f5..13542ff 100644 --- a/src/core/release-evidence.ts +++ b/src/core/release-evidence.ts @@ -161,6 +161,8 @@ export interface BuildDoctorReleaseEvidenceOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; sandbox?: RuntimeSandboxMode; environment?: CompatibilityEnvironment; runCheck?: (targetPath: string, options?: CheckOptions) => Promise; @@ -516,6 +518,8 @@ export async function buildDoctorReleaseEvidenceReport( runtime: true, ...(options.allowNetwork ? { allowNetwork: true } : {}), ...(options.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(options.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(options.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(options.sandbox ? { runtimeSandbox: options.sandbox } : {}) } : {}; diff --git a/src/core/remote-mcp-probe.ts b/src/core/remote-mcp-probe.ts index a19fe22..725582f 100644 --- a/src/core/remote-mcp-probe.ts +++ b/src/core/remote-mcp-probe.ts @@ -6,6 +6,9 @@ import { } from "./bounded-http-client.js"; import { RemoteNetworkPolicyError, type RemoteLookup } from "./remote-network-policy.js"; import { checkRemoteOAuthReadiness } from "./remote-oauth-readiness.js"; +import { + probeRemoteTransportReliability +} from "./remote-transport-reliability.js"; import { inspectRemoteMcpUrl } from "./remote-url-policy.js"; import type { Finding, RemoteRuntimeScorecard, RuntimeCapabilityStatus } from "../domain/types.js"; import { packageVersion } from "../version.js"; @@ -25,6 +28,7 @@ export interface RemoteMcpProbeOptions { requestTimeoutMs?: number; lookup?: RemoteLookup; request?: RemoteMcpRequest; + allowSessionLifecycle?: boolean; } export interface RemoteMcpProbeResult { @@ -54,7 +58,10 @@ function failure( return { id, severity: "fail", message, impact, suggestedFix }; } -function finalize(scorecard: RemoteRuntimeScorecard, findings: Finding[]): RemoteMcpProbeResult { +function finalize( + scorecard: RemoteRuntimeScorecard, + findings: Finding[] +): RemoteMcpProbeResult { scorecard.overall = findings.some((finding) => finding.severity === "fail") ? "fail" : findings.some((finding) => finding.severity === "warn") @@ -133,6 +140,10 @@ function isValidInitializeResponse(message: JsonObject): boolean { ); } +function isInitializedNotificationAcknowledged(response: BoundedHttpResponse): boolean { + return response.statusCode === 202 && response.body.length === 0; +} + function validSessionId(value: string | null): boolean { return value !== null && /^[\x21-\x7e]+$/.test(value); } @@ -305,7 +316,7 @@ export async function probeRemoteMcpServer( ...(sessionId === null ? {} : { "MCP-Session-Id": sessionId }) } }); - if (initializedResponse.statusCode < 200 || initializedResponse.statusCode >= 300) { + if (!isInitializedNotificationAcknowledged(initializedResponse)) { scorecard.protocolHeaders = "fail"; findings.push(failure( "plugin.runtime.remote.initialized.failed", @@ -327,5 +338,59 @@ export async function probeRemoteMcpServer( } scorecard.protocolHeaders = "pass"; + const reliability = await probeRemoteTransportReliability({ + rawUrl, + protocolVersion: MCP_PROTOCOL_VERSION, + request, + requestTimeoutMs: options.requestTimeoutMs, + requestOptions: { + allowLocalNetwork: options.allowLocalNetwork, + lookup: options.lookup + }, + sessionId, + allowSessionLifecycle: options.allowSessionLifecycle, + reinitialize: async (requestWithinBudget) => { + const restartInitialize = await requestWithinBudget({ + method: "POST", + body: initializeBody, + headers: { + Accept: "application/json, text/event-stream", + "Content-Type": "application/json" + }, + stopAfter: (body) => findSseInitializeResponse(body) !== null + }); + if (restartInitialize.statusCode !== 200) { + throw new Error("restart initialize failed"); + } + const restartContentType = mediaType(responseHeader(restartInitialize.headers, "content-type")); + if (restartContentType !== "application/json" && restartContentType !== "text/event-stream") { + throw new Error("restart initialize content type invalid"); + } + const replacementSessionId = responseHeader(restartInitialize.headers, "mcp-session-id"); + if (replacementSessionId !== null && !validSessionId(replacementSessionId)) { + throw new Error("restart session invalid"); + } + const restartMessage = parseInitializeResponse(restartInitialize.body, restartContentType); + if (!restartMessage || !isValidInitializeResponse(restartMessage)) { + throw new Error("restart initialize result invalid"); + } + const restartInitialized = await requestWithinBudget({ + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), + headers: { + Accept: "application/json, text/event-stream", + "Content-Type": "application/json", + "MCP-Protocol-Version": MCP_PROTOCOL_VERSION, + ...(replacementSessionId === null ? {} : { "MCP-Session-Id": replacementSessionId }) + } + }); + if (!isInitializedNotificationAcknowledged(restartInitialized)) { + throw new Error("restart initialized failed"); + } + return replacementSessionId; + } + }); + scorecard.reliability = reliability.scorecard; + findings.push(...reliability.findings); return finalize(scorecard, findings); } diff --git a/src/core/remote-transport-reliability.ts b/src/core/remote-transport-reliability.ts new file mode 100644 index 0000000..596730e --- /dev/null +++ b/src/core/remote-transport-reliability.ts @@ -0,0 +1,363 @@ +import { + BoundedHttpError, + type BoundedHttpRequestOptions, + type BoundedHttpResponse +} from "./bounded-http-client.js"; +import { observeFirstSseEvent, type SseObservation } from "./sse-observation.js"; +import type { Finding, RemoteTransportReliabilityScorecard } from "../domain/types.js"; + +const DEFAULT_RELIABILITY_TIMEOUT_MS = 3_000; +const MAX_RELIABILITY_REQUESTS = 6; + +export interface RemoteTransportReliabilityResult { + findings: Finding[]; + scorecard: RemoteTransportReliabilityScorecard; +} + +export type RemoteTransportReliabilityRequest = ( + options: BoundedHttpRequestOptions +) => Promise; + +export interface RemoteTransportReliabilityOptions { + rawUrl: string; + protocolVersion: string; + request: (rawUrl: string, options?: BoundedHttpRequestOptions) => Promise; + requestTimeoutMs?: number; + requestOptions?: Pick; + sessionId: string | null; + allowSessionLifecycle?: boolean; + reinitialize: (request: RemoteTransportReliabilityRequest) => Promise; +} + +interface RequestResult { + response?: BoundedHttpResponse; + error?: unknown; + restartFailed?: boolean; + sessionRestarted?: boolean; +} + +function createScorecard(): RemoteTransportReliabilityScorecard { + return { + getSse: "skipped", + sessionPropagation: "skipped", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "skipped" + }; +} + +function finding(id: string, severity: "fail" | "warn", message: string, suggestedFix: string): Finding { + return { + id, + severity, + message, + impact: "Remote MCP transport reliability could not be verified within the configured safety bounds.", + suggestedFix + }; +} + +function responseHeader(headers: BoundedHttpResponse["headers"], name: string): string | null { + const value = headers[name]; + return typeof value === "string" ? value : null; +} + +function mediaType(value: string | null): string | null { + return value?.split(";", 1)[0]?.trim().toLowerCase() ?? null; +} + +function validSessionId(value: string | null): boolean { + return value !== null && /^[\x21-\x7e]+$/.test(value); +} + +function wait(delayMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +function finalize(scorecard: RemoteTransportReliabilityScorecard, findings: Finding[]): RemoteTransportReliabilityResult { + const statuses = Object.entries(scorecard) + .filter(([name]) => name !== "overall") + .map(([, status]) => status); + scorecard.overall = statuses.includes("fail") + ? "fail" + : statuses.includes("warn") + ? "warn" + : statuses.includes("pass") + ? "pass" + : "skipped"; + return { findings, scorecard }; +} + +function isSseTimeout(error: unknown): boolean { + return error instanceof BoundedHttpError + && error.code === "REMOTE_HTTP_TIMEOUT" + && error.statusCode === 200 + && mediaType(responseHeader(error.headers ?? {}, "content-type")) === "text/event-stream"; +} + +function classifySseResponse( + response: BoundedHttpResponse, + failurePrefix: "get" | "resume", + findings: Finding[] +): SseObservation | null { + if (response.statusCode !== 200) { + findings.push(finding( + `plugin.runtime.remote.reliability.${failurePrefix}.status`, + "fail", + "The remote MCP endpoint returned an unsupported SSE transport status.", + "Return HTTP 200 for an accepted SSE transport request." + )); + return null; + } + if (mediaType(responseHeader(response.headers, "content-type")) !== "text/event-stream") { + findings.push(finding( + `plugin.runtime.remote.reliability.${failurePrefix}.content_type`, + "fail", + "The remote MCP endpoint returned a non-SSE media type for an SSE transport request.", + "Return text/event-stream for accepted SSE transport requests." + )); + return null; + } + return observeFirstSseEvent(response.body); +} + +export async function probeRemoteTransportReliability( + options: RemoteTransportReliabilityOptions +): Promise { + const scorecard = createScorecard(); + const findings: Finding[] = []; + const timeoutMs = options.requestTimeoutMs ?? DEFAULT_RELIABILITY_TIMEOUT_MS; + const deadline = Date.now() + Math.min(timeoutMs, DEFAULT_RELIABILITY_TIMEOUT_MS); + let currentSessionId = options.sessionId; + let requestCount = 0; + let restarted = false; + + const requestWithinBudget: RemoteTransportReliabilityRequest = async (requestOptions) => { + if (requestCount >= MAX_RELIABILITY_REQUESTS) { + throw new Error("reliability request ceiling reached"); + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "Remote reliability deadline elapsed."); + } + requestCount += 1; + return options.request(options.rawUrl, { + ...requestOptions, + ...options.requestOptions, + timeoutMs: Math.max(1, Math.min(remainingMs, DEFAULT_RELIABILITY_TIMEOUT_MS)) + }); + }; + + const send = async ( + method: "GET" | "DELETE", + extraHeaders: Record = {}, + stopAfter?: (body: Buffer) => boolean + ): Promise => { + let sessionRestarted = false; + for (let attempt = 0; attempt < 2; attempt += 1) { + const sessionId = currentSessionId; + const headers = { + Accept: "text/event-stream", + "MCP-Protocol-Version": options.protocolVersion, + ...(sessionId === null ? {} : { "MCP-Session-Id": sessionId }), + ...(sessionRestarted ? {} : extraHeaders) + }; + if (sessionId !== null) { + scorecard.sessionPropagation = "pass"; + } + let response: BoundedHttpResponse; + try { + response = await requestWithinBudget({ + method, + headers, + stopAfter + }); + } catch (error) { + return { error }; + } + + if (method === "GET" && sessionId !== null && response.statusCode === 404) { + if (restarted) { + scorecard.sessionRestart = "fail"; + return { restartFailed: true }; + } + restarted = true; + sessionRestarted = true; + currentSessionId = null; + try { + const replacementSessionId = await options.reinitialize(requestWithinBudget); + if (replacementSessionId !== null && !validSessionId(replacementSessionId)) { + throw new Error("invalid replacement session"); + } + currentSessionId = replacementSessionId; + scorecard.sessionRestart = "pass"; + } catch { + scorecard.sessionRestart = "fail"; + return { restartFailed: true }; + } + continue; + } + return { response, sessionRestarted }; + } + return { restartFailed: true }; + }; + + const first = await send("GET", {}, (body) => observeFirstSseEvent(body).complete); + if (first.restartFailed) { + findings.push(finding( + "plugin.runtime.remote.reliability.session_restart.failed", + "fail", + "The remote MCP session expired and could not be restarted within the bounded probe.", + "Accept a fresh initialize sequence after an expired MCP session." + )); + return finalize(scorecard, findings); + } + if (first.error) { + if (isSseTimeout(first.error)) { + scorecard.getSse = "warn"; + scorecard.disconnectSafety = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.inconclusive", + "warn", + "The remote MCP SSE stream did not produce a complete event before the bounded observation deadline.", + "Emit complete SSE event frames promptly when server-to-client streaming is supported." + )); + return finalize(scorecard, findings); + } + scorecard.getSse = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.failed", + "fail", + "The remote MCP SSE transport request could not be completed.", + "Keep the Streamable HTTP transport reachable within the configured request bounds." + )); + return finalize(scorecard, findings); + } + const firstResponse = first.response; + if (!firstResponse) { + scorecard.getSse = "fail"; + return finalize(scorecard, findings); + } + if (firstResponse.statusCode === 405) { + scorecard.getSse = "pass"; + } else { + const observation = classifySseResponse(firstResponse, "get", findings); + if (observation === null) { + scorecard.getSse = "fail"; + return finalize(scorecard, findings); + } + if (!observation.complete) { + scorecard.getSse = "warn"; + scorecard.disconnectSafety = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.inconclusive", + "warn", + "The remote MCP SSE response ended before a complete event could be observed.", + "Emit complete SSE event frames when server-to-client streaming is supported." + )); + return finalize(scorecard, findings); + } + if (observation.malformed) { + scorecard.getSse = "fail"; + scorecard.disconnectSafety = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.get.malformed", + "fail", + "The remote MCP endpoint emitted malformed SSE event framing.", + "Return complete SSE events with valid id and retry fields." + )); + return finalize(scorecard, findings); + } + scorecard.getSse = "pass"; + scorecard.disconnectSafety = "pass"; + + if (observation.eventId !== null) { + const remainingMs = deadline - Date.now(); + if (observation.retryMs !== null) { + if (observation.retryMs >= remainingMs) { + scorecard.resumability = "warn"; + findings.push(finding( + "plugin.runtime.remote.reliability.resume.inconclusive", + "warn", + "The remote MCP SSE retry delay exceeded the remaining bounded observation deadline.", + "Advertise an SSE retry delay that fits within the configured probe deadline." + )); + return finalize(scorecard, findings); + } + await wait(observation.retryMs); + } + const resumed = await send("GET", { "Last-Event-ID": observation.eventId }, (body) => observeFirstSseEvent(body).complete); + if (resumed.restartFailed) { + findings.push(finding( + "plugin.runtime.remote.reliability.session_restart.failed", + "fail", + "The remote MCP session expired during SSE resumability validation and could not be restarted.", + "Accept one fresh initialize sequence after an expired MCP session." + )); + return finalize(scorecard, findings); + } + if (resumed.error) { + scorecard.resumability = isSseTimeout(resumed.error) ? "warn" : "fail"; + findings.push(finding( + isSseTimeout(resumed.error) + ? "plugin.runtime.remote.reliability.resume.inconclusive" + : "plugin.runtime.remote.reliability.resume.failed", + isSseTimeout(resumed.error) ? "warn" : "fail", + "The remote MCP SSE resume request could not be completed within the bounded probe.", + "Accept one bounded SSE reconnect using Last-Event-ID." + )); + return finalize(scorecard, findings); + } + if (!resumed.response) { + scorecard.resumability = "fail"; + return finalize(scorecard, findings); + } + if (resumed.sessionRestarted) { + scorecard.resumability = "skipped"; + } else { + const resumeObservation = classifySseResponse(resumed.response, "resume", findings); + if (resumeObservation === null || resumeObservation.malformed) { + scorecard.resumability = "fail"; + if (resumeObservation?.malformed) { + findings.push(finding( + "plugin.runtime.remote.reliability.resume.malformed", + "fail", + "The remote MCP endpoint emitted malformed SSE framing after a resume request.", + "Return complete SSE events with valid id and retry fields after reconnecting." + )); + } + return finalize(scorecard, findings); + } + scorecard.resumability = resumeObservation.complete ? "pass" : "warn"; + if (!resumeObservation.complete) { + findings.push(finding( + "plugin.runtime.remote.reliability.resume.inconclusive", + "warn", + "The remote MCP SSE resume response ended before a complete event could be observed.", + "Emit complete SSE events after accepting a reconnect." + )); + return finalize(scorecard, findings); + } + } + } + } + + if (!options.allowSessionLifecycle || currentSessionId === null) { + return finalize(scorecard, findings); + } + const terminated = await send("DELETE"); + if (terminated.restartFailed || terminated.error || !terminated.response + || (terminated.response.statusCode !== 405 && (terminated.response.statusCode < 200 || terminated.response.statusCode >= 300))) { + scorecard.termination = "fail"; + findings.push(finding( + "plugin.runtime.remote.reliability.termination.failed", + "fail", + "The remote MCP session could not be terminated with the approved lifecycle request.", + "Return a successful response or HTTP 405 for a bounded MCP session DELETE request." + )); + return finalize(scorecard, findings); + } + scorecard.termination = "pass"; + return finalize(scorecard, findings); +} diff --git a/src/core/runtime-probe.ts b/src/core/runtime-probe.ts index b851312..de5fc65 100644 --- a/src/core/runtime-probe.ts +++ b/src/core/runtime-probe.ts @@ -539,6 +539,23 @@ function mergeRemoteRuntimeScorecards( "present-valid": 1, absent: 0 }; + const reliability = left.reliability && right.reliability + ? { + getSse: worstRuntimeStatus(left.reliability.getSse, right.reliability.getSse), + sessionPropagation: worstRuntimeStatus( + left.reliability.sessionPropagation, + right.reliability.sessionPropagation + ), + resumability: worstRuntimeStatus(left.reliability.resumability, right.reliability.resumability), + disconnectSafety: worstRuntimeStatus( + left.reliability.disconnectSafety, + right.reliability.disconnectSafety + ), + sessionRestart: worstRuntimeStatus(left.reliability.sessionRestart, right.reliability.sessionRestart), + termination: worstRuntimeStatus(left.reliability.termination, right.reliability.termination), + overall: worstRuntimeStatus(left.reliability.overall, right.reliability.overall) + } + : left.reliability ?? right.reliability; return { transport: worstRuntimeStatus(left.transport, right.transport), @@ -550,7 +567,8 @@ function mergeRemoteRuntimeScorecards( : right.session, protocolHeaders: worstRuntimeStatus(left.protocolHeaders, right.protocolHeaders), authorization: worstRuntimeStatus(left.authorization, right.authorization), - overall: worstRuntimeStatus(left.overall, right.overall) + overall: worstRuntimeStatus(left.overall, right.overall), + ...(reliability ? { reliability } : {}) }; } @@ -1975,11 +1993,17 @@ export interface RuntimeProbeOptions { transcript?: (line: string) => void; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; remoteRequestTimeoutMs?: number; remoteLookup?: RemoteLookup; remoteRequest?: RemoteMcpRequest; } +export function remoteReliabilityGatePassed(scorecard: RuntimeScorecard | undefined): boolean { + return scorecard !== undefined && + (scorecard.remote === undefined || scorecard.remote.reliability?.overall === "pass"); +} + export async function probeRuntimeConfig( rootPath: string, mcpConfigPath: string, @@ -2036,6 +2060,7 @@ export async function probeRuntimeConfig( const remote = await probeRemoteMcpServer(serverName, url, { allowNetwork: options.allowNetwork, allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle, requestTimeoutMs: options.remoteRequestTimeoutMs, lookup: options.remoteLookup, request: options.remoteRequest diff --git a/src/core/sse-observation.ts b/src/core/sse-observation.ts new file mode 100644 index 0000000..643186d --- /dev/null +++ b/src/core/sse-observation.ts @@ -0,0 +1,82 @@ +const MAX_SSE_FIELD_BYTES = 1_024; + +export interface SseObservation { + complete: boolean; + eventId: string | null; + retryMs: number | null; + malformed: boolean; +} + +function isSafeEventId(value: Buffer): boolean { + return value.length > 0 + && value.length <= MAX_SSE_FIELD_BYTES + && value.every((byte) => byte >= 0x21 && byte <= 0x7e); +} + +function parseRetry(value: Buffer): number | null { + if (value.length === 0 || value.length > MAX_SSE_FIELD_BYTES) { + return null; + } + + for (const byte of value) { + if (byte < 0x30 || byte > 0x39) { + return null; + } + } + + const retryMs = Number(value.toString("ascii")); + return Number.isSafeInteger(retryMs) ? retryMs : null; +} + +export function observeFirstSseEvent(body: Buffer): SseObservation { + let eventId: string | null = null; + let retryMs: number | null = null; + let malformed = false; + let lineStart = 0; + + for (let index = 0; index < body.length; index += 1) { + if (body[index] !== 0x0a && body[index] !== 0x0d) { + continue; + } + + const line = body.subarray(lineStart, index); + if (body[index] === 0x0d && body[index + 1] === 0x0a) { + index += 1; + } + lineStart = index + 1; + + if (line.length === 0) { + return { + complete: true, + eventId: malformed ? null : eventId, + retryMs: malformed ? null : retryMs, + malformed + }; + } + if (line[0] === 0x3a) { + continue; + } + + const separator = line.indexOf(0x3a); + const field = separator === -1 ? line : line.subarray(0, separator); + const valueStart = separator === -1 ? line.length : separator + 1; + const value = line.subarray(valueStart + (line[valueStart] === 0x20 ? 1 : 0)); + + if (field.equals(Buffer.from("id"))) { + if (value.length === 0) { + eventId = null; + } else if (isSafeEventId(value)) { + eventId = value.toString("ascii"); + } else { + malformed = true; + } + } else if (field.equals(Buffer.from("retry"))) { + const parsedRetryMs = parseRetry(value); + if (parsedRetryMs !== null) { + retryMs = parsedRetryMs; + } + } + } + + return { complete: false, eventId: null, retryMs: null, malformed: false }; +} diff --git a/src/core/validate-plugin.ts b/src/core/validate-plugin.ts index 0cee031..82bb516 100644 --- a/src/core/validate-plugin.ts +++ b/src/core/validate-plugin.ts @@ -11,7 +11,7 @@ import type { import { withFindingFingerprints } from "../reporting/finding-fingerprint.js"; import { discoverPackage } from "./discover-package.js"; import { inspectRemoteMcpUrl } from "./remote-url-policy.js"; -import { probeRuntime } from "./runtime-probe.js"; +import { probeRuntime, remoteReliabilityGatePassed } from "./runtime-probe.js"; function buildFailure( id: string, @@ -791,7 +791,8 @@ export async function validatePlugin( sandbox: options.runtimeSandbox, transcript: options.runtimeTranscript, allowNetwork: options.allowNetwork, - allowLocalNetwork: options.allowLocalNetwork + allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle }) : null; const findings = [ @@ -805,7 +806,7 @@ export async function validatePlugin( ); const hasFailures = fingerprintedFindings.some( (finding) => finding.severity === "fail" - ); + ) || (options.requireRemoteReliability === true && !remoteReliabilityGatePassed(runtimeResult?.scorecard)); const hasWarnings = fingerprintedFindings.some( (finding) => finding.severity === "warn" ); diff --git a/src/domain/types.ts b/src/domain/types.ts index e7ab7d4..24db339 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -52,6 +52,8 @@ export interface CheckOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeTranscript?: (line: string) => void; runtimeStartupTimeoutMs?: number; runtimeSandbox?: RuntimeSandboxMode; @@ -124,9 +126,22 @@ export interface RemoteRuntimeScorecard { session: "absent" | "present-valid" | "present-invalid"; protocolHeaders: RuntimeCapabilityStatus; authorization: RuntimeCapabilityStatus; + reliability?: RemoteTransportReliabilityScorecard; overall: "pass" | "warn" | "fail" | "skipped"; } +export type RemoteTransportReliabilityStatus = "pass" | "warn" | "fail" | "skipped"; + +export interface RemoteTransportReliabilityScorecard { + getSse: RemoteTransportReliabilityStatus; + sessionPropagation: RemoteTransportReliabilityStatus; + resumability: RemoteTransportReliabilityStatus; + disconnectSafety: RemoteTransportReliabilityStatus; + sessionRestart: RemoteTransportReliabilityStatus; + termination: RemoteTransportReliabilityStatus; + overall: RemoteTransportReliabilityStatus; +} + export type McpConformanceProfile = | "legacy" | "2025-11-25" diff --git a/src/mcp/generic-mcp-doctor.ts b/src/mcp/generic-mcp-doctor.ts index 1a18166..c140bcb 100644 --- a/src/mcp/generic-mcp-doctor.ts +++ b/src/mcp/generic-mcp-doctor.ts @@ -8,7 +8,7 @@ import { readMcpConfigPath } from "../compatibility/compatibility-matrix.js"; import { readJsonFile } from "../core/read-json-file.js"; -import { probeRuntimeConfig } from "../core/runtime-probe.js"; +import { probeRuntimeConfig, remoteReliabilityGatePassed } from "../core/runtime-probe.js"; import type { Finding, FindingEvidence, @@ -42,6 +42,8 @@ export interface GenericMcpDoctorOptions { runtime?: boolean; allowNetwork?: boolean; allowLocalNetwork?: boolean; + allowSessionLifecycle?: boolean; + requireRemoteReliability?: boolean; runtimeStartupTimeoutMs?: number; } @@ -289,14 +291,17 @@ export async function buildGenericMcpDoctor( ? await probeRuntimeConfig(canonicalRootPath, canonicalMcpConfigPath, { startupTimeoutMs: options.runtimeStartupTimeoutMs, allowNetwork: options.allowNetwork, - allowLocalNetwork: options.allowLocalNetwork + allowLocalNetwork: options.allowLocalNetwork, + allowSessionLifecycle: options.allowSessionLifecycle }) : null; const fingerprintedFindings = withFindingFingerprints( [...staticFindings, ...(runtimeResult?.findings ?? [])], rootPath ); - const status = mergeReportStatus(fingerprintedFindings, security); + const status = options.requireRemoteReliability === true && !remoteReliabilityGatePassed(runtimeResult?.scorecard) + ? "fail" + : mergeReportStatus(fingerprintedFindings, security); return { targetPath: rootPath, diff --git a/src/reporting/render-markdown-report.ts b/src/reporting/render-markdown-report.ts index d7a7700..00b199c 100644 --- a/src/reporting/render-markdown-report.ts +++ b/src/reporting/render-markdown-report.ts @@ -49,6 +49,20 @@ function appendRuntimeScorecard(lines: string[], result: CheckResult) { lines.push(`| Protocol headers | ${remote.protocolHeaders.toUpperCase()} |`); lines.push(`| Authorization | ${remote.authorization.toUpperCase()} |`); lines.push(`| Overall | ${remote.overall.toUpperCase()} |`); + + if (remote.reliability) { + const reliability = remote.reliability; + lines.push("", "## Remote Transport Reliability", ""); + lines.push("| Check | Status |"); + lines.push("| --- | --- |"); + lines.push(`| GET SSE | ${reliability.getSse.toUpperCase()} |`); + lines.push(`| Session propagation | ${reliability.sessionPropagation.toUpperCase()} |`); + lines.push(`| Resumability | ${reliability.resumability.toUpperCase()} |`); + lines.push(`| Disconnect safety | ${reliability.disconnectSafety.toUpperCase()} |`); + lines.push(`| Session restart | ${reliability.sessionRestart.toUpperCase()} |`); + lines.push(`| Termination | ${reliability.termination.toUpperCase()} |`); + lines.push(`| Overall | ${reliability.overall.toUpperCase()} |`); + } } } diff --git a/src/reporting/render-text-report.ts b/src/reporting/render-text-report.ts index e79153d..0e2b176 100644 --- a/src/reporting/render-text-report.ts +++ b/src/reporting/render-text-report.ts @@ -73,6 +73,18 @@ function appendRuntimeScorecard(lines: string[], result: CheckResult) { lines.push(`protocol headers: ${remote.protocolHeaders}`); lines.push(`authorization: ${remote.authorization}`); lines.push(`overall: ${remote.overall}`); + + if (remote.reliability) { + const reliability = remote.reliability; + lines.push("", "Remote Transport Reliability", "----------------------------"); + lines.push(`GET SSE: ${reliability.getSse}`); + lines.push(`Session propagation: ${reliability.sessionPropagation}`); + lines.push(`Resumability: ${reliability.resumability}`); + lines.push(`Disconnect safety: ${reliability.disconnectSafety}`); + lines.push(`Session restart: ${reliability.sessionRestart}`); + lines.push(`Termination: ${reliability.termination}`); + lines.push(`Overall: ${reliability.overall}`); + } } } diff --git a/src/rules/rule-catalog.ts b/src/rules/rule-catalog.ts index c0411b7..18c7af3 100644 --- a/src/rules/rule-catalog.ts +++ b/src/rules/rule-catalog.ts @@ -575,6 +575,114 @@ export const ruleCatalog: RuleDefinition[] = [ why: "Authorization readiness cannot be confirmed without bounded metadata responses.", fix: "Make protected-resource and authorization-server metadata available over HTTPS.", example: "Return application/json discovery metadata." + }, + { + id: "plugin.runtime.remote.reliability.get.status", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint rejected the SSE transport request.", + why: "Clients cannot rely on server-to-client streaming when an accepted SSE request does not return the expected status.", + fix: "Return HTTP 200 with a valid SSE response, or HTTP 405 when GET streaming is not supported.", + example: "HTTP/1.1 405 Method Not Allowed" + }, + { + id: "plugin.runtime.remote.reliability.get.content_type", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint returned a non-SSE media type for GET streaming.", + why: "Clients cannot interpret a successful streaming response unless it is declared as Server-Sent Events.", + fix: "Return Content-Type: text/event-stream for accepted GET streaming requests.", + example: "Content-Type: text/event-stream" + }, + { + id: "plugin.runtime.remote.reliability.get.inconclusive", + category: "runtime", + defaultSeverity: "warn", + summary: "The remote MCP GET streaming check was inconclusive.", + why: "A bounded probe could not observe a complete SSE event, so streaming reliability remains uncertain.", + fix: "Emit complete SSE event frames promptly when server-to-client streaming is supported.", + example: "id: event-1\n\ndata: {}\n\n" + }, + { + id: "plugin.runtime.remote.reliability.get.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP GET streaming request could not be completed.", + why: "Clients cannot rely on server-to-client streaming when a bounded transport request fails.", + fix: "Keep the Streamable HTTP transport reachable within the configured request bounds.", + example: "Complete the GET request before the runtime timeout." + }, + { + id: "plugin.runtime.remote.reliability.get.malformed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint emitted malformed SSE framing.", + why: "Malformed event framing prevents clients from safely processing stream progress or reconnecting.", + fix: "Return complete SSE events with valid id and retry fields.", + example: "id: event-1\n\nevent: message\n\ndata: {}\n\n" + }, + { + id: "plugin.runtime.remote.reliability.resume.status", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint rejected the SSE resume request.", + why: "Clients cannot resume a dropped stream when the reconnect response has an unexpected status.", + fix: "Return HTTP 200 for an accepted Last-Event-ID reconnect request.", + example: "HTTP/1.1 200 OK" + }, + { + id: "plugin.runtime.remote.reliability.resume.content_type", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint returned a non-SSE media type for a resume request.", + why: "Clients cannot process a resumed stream unless it is declared as Server-Sent Events.", + fix: "Return Content-Type: text/event-stream after accepting a Last-Event-ID reconnect request.", + example: "Content-Type: text/event-stream" + }, + { + id: "plugin.runtime.remote.reliability.resume.inconclusive", + category: "runtime", + defaultSeverity: "warn", + summary: "The remote MCP SSE resume check was inconclusive.", + why: "A bounded probe could not confirm a complete resumed event, so reconnect reliability remains uncertain.", + fix: "Emit complete SSE event frames after accepting a reconnect.", + example: "Return one complete SSE event after a Last-Event-ID request." + }, + { + id: "plugin.runtime.remote.reliability.resume.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP SSE resume request could not be completed.", + why: "Clients cannot recover from a dropped stream when a bounded reconnect fails.", + fix: "Accept one bounded SSE reconnect using Last-Event-ID.", + example: "Honor the Last-Event-ID header on one reconnect." + }, + { + id: "plugin.runtime.remote.reliability.resume.malformed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP endpoint emitted malformed SSE framing after reconnecting.", + why: "Malformed resumed events prevent clients from safely continuing stream processing.", + fix: "Return complete SSE events with valid id and retry fields after reconnecting.", + example: "Return a complete event frame after reconnecting." + }, + { + id: "plugin.runtime.remote.reliability.session_restart.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP session could not be restarted.", + why: "Clients cannot recover from an expired MCP session when a fresh initialization sequence fails.", + fix: "Accept a fresh initialize sequence after an expired MCP session.", + example: "Return a valid initialize response after HTTP 404 for a session-bound request." + }, + { + id: "plugin.runtime.remote.reliability.termination.failed", + category: "runtime", + defaultSeverity: "fail", + summary: "The remote MCP session could not be terminated.", + why: "A server that advertises session lifecycle support must safely handle the approved termination request.", + fix: "Return a successful response or HTTP 405 for a bounded MCP session DELETE request.", + example: "HTTP/1.1 204 No Content" } ]; diff --git a/src/run-cli.ts b/src/run-cli.ts index 6c78436..d059fdb 100644 --- a/src/run-cli.ts +++ b/src/run-cli.ts @@ -324,9 +324,16 @@ function parseRuntimeSandbox( function parseRemoteNetworkFlags( flags: string[], runtime: boolean -): { allowNetwork: boolean; allowLocalNetwork: boolean } | CliUsageError { +): { + allowNetwork: boolean; + allowLocalNetwork: boolean; + allowSessionLifecycle: boolean; + requireRemoteReliability: boolean; +} | CliUsageError { let allowNetwork = false; let allowLocalNetwork = false; + let allowSessionLifecycle = false; + let requireRemoteReliability = false; for (let index = 0; index < flags.length; index += 1) { const flag = flags[index]; @@ -343,11 +350,36 @@ function parseRemoteNetworkFlags( return new CliUsageError("--allow-local-network does not accept a value."); } allowLocalNetwork = true; - } else if (flag?.startsWith("--allow-network=") || flag?.startsWith("--allow-local-network=")) { + } else if (flag === "--allow-session-lifecycle") { + if (allowSessionLifecycle) return new CliUsageError("Duplicate runtime remote flag: --allow-session-lifecycle."); + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return new CliUsageError("--allow-session-lifecycle does not accept a value."); + } + allowSessionLifecycle = true; + } else if (flag === "--require-remote-reliability") { + if (requireRemoteReliability) return new CliUsageError("Duplicate runtime remote flag: --require-remote-reliability."); + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return new CliUsageError("--require-remote-reliability does not accept a value."); + } + requireRemoteReliability = true; + } else if ( + flag?.startsWith("--allow-network=") || + flag?.startsWith("--allow-local-network=") || + flag?.startsWith("--allow-session-lifecycle=") || + flag?.startsWith("--require-remote-reliability=") + ) { return new CliUsageError(`${flag.split("=", 1)[0]} does not accept a value.`); } } + if (allowSessionLifecycle && !runtime) { + return new CliUsageError("--allow-session-lifecycle requires --runtime."); + } + + if (requireRemoteReliability && !runtime) { + return new CliUsageError("--require-remote-reliability requires --runtime and --allow-network."); + } + if ((allowNetwork || allowLocalNetwork) && !runtime) { return new CliUsageError("--allow-network requires --runtime."); } @@ -356,7 +388,15 @@ function parseRemoteNetworkFlags( return new CliUsageError("--allow-local-network requires --allow-network."); } - return { allowNetwork, allowLocalNetwork }; + if (allowSessionLifecycle && !allowNetwork) { + return new CliUsageError("--allow-session-lifecycle requires --allow-network."); + } + + if (requireRemoteReliability && !allowNetwork) { + return new CliUsageError("--require-remote-reliability requires --runtime and --allow-network."); + } + + return { allowNetwork, allowLocalNetwork, allowSessionLifecycle, requireRemoteReliability }; } function printUsage(io: CliIo): void { @@ -372,6 +412,9 @@ function printUsage(io: CliIo): void { io.writeStderr( "Baseline gating: codex-plugin-doctor baseline create --output [--runtime]\n codex-plugin-doctor check --baseline " ); + io.writeStderr( + "Remote MCP runtime flags (check, mcp, release check, and doctor release-evidence): --runtime --allow-network [--allow-local-network] [--allow-session-lifecycle] [--require-remote-reliability]\n --allow-session-lifecycle requires --runtime --allow-network and can terminate a remote session.\n --require-remote-reliability requires --runtime --allow-network, blocks non-pass reliability, and does not grant network consent." + ); } const suppressUsageText = [ @@ -1185,6 +1228,8 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] runtime: boolean; allowNetwork: boolean; allowLocalNetwork: boolean; + allowSessionLifecycle: boolean; + requireRemoteReliability: boolean; } | string { if (!commandTarget || commandTarget.startsWith("--")) { return "Missing target path. Usage: codex-plugin-doctor mcp [--runtime] [--json] [--output ]"; @@ -1195,6 +1240,8 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] let runtime = false; let allowNetwork = false; let allowLocalNetwork = false; + let allowSessionLifecycle = false; + let requireRemoteReliability = false; for (let index = 0; index < flags.length; index += 1) { const flag = flags[index]; @@ -1229,6 +1276,31 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] continue; } + if (flag === "--allow-session-lifecycle") { + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return "--allow-session-lifecycle does not accept a value."; + } + allowSessionLifecycle = true; + continue; + } + + if (flag === "--require-remote-reliability") { + if (flags[index + 1] && !flags[index + 1]!.startsWith("--")) { + return "--require-remote-reliability does not accept a value."; + } + requireRemoteReliability = true; + continue; + } + + if ( + flag.startsWith("--allow-network=") || + flag.startsWith("--allow-local-network=") || + flag.startsWith("--allow-session-lifecycle=") || + flag.startsWith("--require-remote-reliability=") + ) { + return `${flag.split("=", 1)[0]} does not accept a value.`; + } + if (flag === "--output") { if (outputPath !== null) { return "Duplicate MCP flag: --output."; @@ -1261,8 +1333,10 @@ function buildGenericMcpDoctorCommandArgs(commandTarget: string, flags: string[] jsonOutput, outputPath, runtime, - allowNetwork, - allowLocalNetwork + allowNetwork: remoteNetwork.allowNetwork, + allowLocalNetwork: remoteNetwork.allowLocalNetwork, + allowSessionLifecycle, + requireRemoteReliability }; } @@ -1654,7 +1728,9 @@ export async function runCli( }, { runtime: parsedMcpArgs.runtime, allowNetwork: parsedMcpArgs.allowNetwork, - allowLocalNetwork: parsedMcpArgs.allowLocalNetwork + allowLocalNetwork: parsedMcpArgs.allowLocalNetwork, + allowSessionLifecycle: parsedMcpArgs.allowSessionLifecycle, + requireRemoteReliability: parsedMcpArgs.requireRemoteReliability }); const renderedReport = parsedMcpArgs.jsonOutput ? renderGenericMcpDoctorJson(report) @@ -2045,6 +2121,8 @@ export async function runCli( runtime, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { sandbox: runtimeSandbox } : {}), environment: { env: terminalContext.env, @@ -2240,6 +2318,8 @@ export async function runCli( runtime, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { sandbox: runtimeSandbox } : {}), environment: { env: terminalContext.env, @@ -3262,7 +3342,9 @@ export async function runCli( }, { runtime: parsedMcpArgs.runtime, allowNetwork: parsedMcpArgs.allowNetwork, - allowLocalNetwork: parsedMcpArgs.allowLocalNetwork + allowLocalNetwork: parsedMcpArgs.allowLocalNetwork, + allowSessionLifecycle: parsedMcpArgs.allowSessionLifecycle, + requireRemoteReliability: parsedMcpArgs.requireRemoteReliability }); const renderedReport = parsedMcpArgs.jsonOutput ? renderGenericMcpDoctorJson(report) @@ -3659,6 +3741,8 @@ export async function runCli( runtime: runtimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { runtimeSandbox } : {}), runCheck: options.runCheckImpl }); @@ -3932,6 +4016,8 @@ export async function runCli( runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), runtimeTranscript: effectiveRuntimeProbeEnabled && verboseRuntime ? (line) => io.writeStderr(line) @@ -4034,7 +4120,9 @@ export async function runCli( await runCheckImpl(pluginRoot, { runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), - ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}) + ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}) }), applyPolicyToDoctorConfig( applyCheckProfile(await loadDoctorConfig(pluginRoot, configPath), checkProfile), @@ -4070,6 +4158,8 @@ export async function runCli( runtime: effectiveRuntimeProbeEnabled, ...(remoteNetwork.allowNetwork ? { allowNetwork: true } : {}), ...(remoteNetwork.allowLocalNetwork ? { allowLocalNetwork: true } : {}), + ...(remoteNetwork.allowSessionLifecycle ? { allowSessionLifecycle: true } : {}), + ...(remoteNetwork.requireRemoteReliability ? { requireRemoteReliability: true } : {}), ...(runtimeSandbox ? { runtimeSandbox } : {}), ...(effectiveRuntimeProbeEnabled && verboseRuntime ? { runtimeTranscript: (line: string) => io.writeStderr(line) } diff --git a/tests/action-metadata.test.ts b/tests/action-metadata.test.ts index 4dc3cdb..af356f7 100644 --- a/tests/action-metadata.test.ts +++ b/tests/action-metadata.test.ts @@ -102,6 +102,17 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).not.toContain('args+=(--allow-local-network "${{ inputs'); }); + it("forwards explicit lifecycle consent and strict reliability gating through every Action output", async () => { + const actionMetadata = await readFile("action.yml", "utf8"); + + expect(actionMetadata).toMatch(/allow-session-lifecycle:[\s\S]*?default: "false"/); + expect(actionMetadata).toMatch(/require-remote-reliability:[\s\S]*?default: "false"/); + expect(actionMetadata).toContain('ALLOW_SESSION_LIFECYCLE_INPUT: ${{ inputs[\'allow-session-lifecycle\'] }}'); + expect(actionMetadata).toContain('REQUIRE_REMOTE_RELIABILITY_INPUT: ${{ inputs[\'require-remote-reliability\'] }}'); + expect(actionMetadata).toContain('args+=(--allow-session-lifecycle)'); + expect(actionMetadata).toContain('args+=(--require-remote-reliability)'); + }); + it("documents loopback-only consent without permitting private or reserved ranges", async () => { const actionMetadata = await readFile("action.yml", "utf8"); const actionUsage = await readFile("docs/guides/github-action.md", "utf8"); @@ -112,6 +123,8 @@ describe("GitHub Action metadata", () => { expect(document).toContain("loopback endpoints only"); expect(document).toContain("Private, link-local, multicast, unspecified, reserved, and NAT64 ranges remain blocked."); } + expect(actionUsage).toContain('require-remote-reliability: "true"'); + expect(actionUsage).toContain('allow-session-lifecycle: "false"'); }); it("documents the public GitHub Action consumer workflow", async () => { diff --git a/tests/bounded-http-client.test.ts b/tests/bounded-http-client.test.ts index c0cde8d..9dbbbf6 100644 --- a/tests/bounded-http-client.test.ts +++ b/tests/bounded-http-client.test.ts @@ -128,6 +128,18 @@ describe("requestBoundedHttp", () => { })).resolves.toMatchObject({ body: Buffer.from("ok") }); }); + it("allows last-event-id without widening the request header allowlist", async () => { + const port = await startServer((request, response) => { + expect(request.headers["last-event-id"]).toBe("event-1"); + response.end("ok"); + }); + + await expect(requestBoundedHttp(options(port).url, { + ...options(port), + headers: { "Last-Event-ID": "event-1" } + })).resolves.toMatchObject({ body: Buffer.from("ok") }); + }); + it.each([ ["timeoutMs", 0], ["timeoutMs", -1], @@ -259,13 +271,74 @@ describe("requestBoundedHttp", () => { await expect(responseClosed).resolves.toBeUndefined(); }); - it("times out a non-responsive request", async () => { - const port = await startServer(() => undefined); + it("times out without response metadata before response headers arrive", async () => { + vi.useFakeTimers(); + const request = new EventEmitter(); + const requestEnd = vi.fn(); + Object.assign(request, { destroy: vi.fn(), end: requestEnd }); + requestMock.mockReturnValueOnce(request as unknown as ClientRequest); - await expect(requestBoundedHttp(options(port).url, { ...options(port), timeoutMs: 20 })).rejects.toMatchObject({ - code: "REMOTE_HTTP_TIMEOUT", - message: "Remote HTTP request timed out." + try { + const pending = requestBoundedHttp("http://mcp.test/mcp", { + allowLocalNetwork: true, + lookup: localLookup(), + timeoutMs: 20 + }); + await vi.advanceTimersByTimeAsync(0); + expect(requestEnd).toHaveBeenCalledOnce(); + const timedOut = expect(pending).rejects.toMatchObject({ + code: "REMOTE_HTTP_TIMEOUT", + message: "Remote HTTP request timed out.", + statusCode: undefined, + headers: undefined + }); + await vi.advanceTimersByTimeAsync(20); + + await timedOut; + } finally { + vi.useRealTimers(); + } + }); + + it("retains only safe response metadata when a response times out after headers arrive", async () => { + vi.useFakeTimers(); + const response = new EventEmitter(); + Object.assign(response, { + destroy: vi.fn(), + headers: { + "mcp-session-id": "session-1", + "set-cookie": "secret=value", + "x-internal": "hidden" + }, + socket: { remoteAddress: "127.0.0.1" }, + statusCode: 200 + }); + const request = new EventEmitter(); + Object.assign(request, { + destroy: vi.fn(), + end: () => request.emit("response", response) }); + requestMock.mockReturnValueOnce(request as unknown as ClientRequest); + + try { + const pending = requestBoundedHttp("http://mcp.test/mcp", { + allowLocalNetwork: true, + lookup: localLookup(), + timeoutMs: 20 + }); + await vi.advanceTimersByTimeAsync(0); + const timedOut = expect(pending).rejects.toMatchObject({ + code: "REMOTE_HTTP_TIMEOUT", + message: "Remote HTTP request timed out.", + statusCode: 200, + headers: { "mcp-session-id": "session-1" } + }); + await vi.advanceTimersByTimeAsync(20); + + await timedOut; + } finally { + vi.useRealTimers(); + } }); it("uses the DNS-pinned local target", async () => { diff --git a/tests/check-command.test.ts b/tests/check-command.test.ts index baf0abc..52b9105 100644 --- a/tests/check-command.test.ts +++ b/tests/check-command.test.ts @@ -4,6 +4,22 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { runCheck } from "../src/index.js"; +import { remoteReliabilityGatePassed } from "../src/core/runtime-probe.js"; +import type { RuntimeScorecard } from "../src/domain/types.js"; + +function runtimeScorecard(remote?: RuntimeScorecard["remote"]): RuntimeScorecard { + return { + initialize: "skipped", + toolsList: "skipped", + toolsCall: "skipped", + resourcesList: "skipped", + resourceRead: "skipped", + resourceTemplatesList: "skipped", + promptsList: "skipped", + promptGet: "skipped", + ...(remote ? { remote } : {}) + }; +} async function createPluginWithMissingSkillFiles(): Promise { const targetPath = await mkdtemp( @@ -28,6 +44,25 @@ async function createPluginWithMissingSkillFiles(): Promise { } describe("runCheck", () => { + it.each(["warn", "fail", "skipped"] as const)("rejects a %s remote reliability result at the strict gate", (overall) => { + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: overall, sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "pass", sessionRestart: "skipped", termination: "skipped", overall } + }))).toBe(false); + }); + + it("accepts a passing remote reliability result and ignores local-only runtime scorecards", () => { + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "pass", sessionRestart: "skipped", termination: "skipped", overall: "pass" } + }))).toBe(true); + expect(remoteReliabilityGatePassed(runtimeScorecard())).toBe(true); + expect(remoteReliabilityGatePassed(runtimeScorecard({ + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "absent", protocolHeaders: "pass", authorization: "skipped", overall: "pass" + }))).toBe(false); + expect(remoteReliabilityGatePassed(undefined)).toBe(false); + }); + it("distinguishes repeated skill findings with package-relative evidence", async () => { const targetPath = await createPluginWithMissingSkillFiles(); diff --git a/tests/cli-command.test.ts b/tests/cli-command.test.ts index 1dc7051..68a2ef4 100644 --- a/tests/cli-command.test.ts +++ b/tests/cli-command.test.ts @@ -219,6 +219,10 @@ describe("runCli", () => { ["--runtme"], ["unexpected"], ["--runtime", "--runtime"], + ["--runtime", "--allow-network", "--allow-session-lifecycle", "--allow-session-lifecycle"], + ["--runtime", "--allow-network", "--require-remote-reliability", "--require-remote-reliability"], + ["--runtime", "--allow-network", "--allow-session-lifecycle", "true"], + ["--runtime", "--allow-network", "--require-remote-reliability", "true"], ["--json", "--json"], ["--output"], ["--output", "--json"], @@ -3015,6 +3019,67 @@ describe("runCli", () => { }); }); + it("documents remote lifecycle consent and reliability gate dependencies in general help", async () => { + const { io, stderr } = createIo(); + + expect(await runCli([], io)).toBe(2); + + const usage = stderr.join(""); + expect(usage).toContain("--allow-session-lifecycle"); + expect(usage).toContain("--require-remote-reliability"); + expect(usage).toContain("can terminate a remote session"); + expect(usage).toContain("requires --runtime --allow-network"); + expect(usage).toContain("does not grant network consent"); + }); + + it("propagates explicit lifecycle consent and strict remote reliability gating to a runtime check", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + const exitCode = await runCli( + [ + "check", + "tests/fixtures/valid-plugin", + "--runtime", + "--allow-network", + "--allow-session-lifecycle", + "--require-remote-reliability", + "--json" + ], + io, + { runCheckImpl } + ); + + expect(exitCode).toBe(0); + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + + it.each([ + [["--allow-session-lifecycle"], "--allow-session-lifecycle requires --runtime."], + [["--runtime", "--allow-session-lifecycle"], "--allow-session-lifecycle requires --allow-network."], + [["--runtime", "--require-remote-reliability"], "--require-remote-reliability requires --runtime and --allow-network."], + [["--runtime", "--allow-network", "--allow-session-lifecycle", "--allow-session-lifecycle"], "Duplicate runtime remote flag: --allow-session-lifecycle."], + [["--runtime", "--allow-network", "--require-remote-reliability", "--require-remote-reliability"], "Duplicate runtime remote flag: --require-remote-reliability."] + ])("rejects invalid lifecycle and reliability runtime flag combinations", async (flags, message) => { + const { io, stderr } = createIo(); + + const exitCode = await runCli(["check", "tests/fixtures/valid-plugin", ...flags], io); + + expect(exitCode).toBe(2); + expect(stderr.join("")).toContain(message); + }); + it("initializes a minimal Codex plugin package", async () => { const targetPath = await mkdtemp(path.join(os.tmpdir(), "codex-plugin-init-")); const { io, stdout, stderr } = createIo(); diff --git a/tests/json-runtime-scorecard.test.ts b/tests/json-runtime-scorecard.test.ts index 4673daf..21c659f 100644 --- a/tests/json-runtime-scorecard.test.ts +++ b/tests/json-runtime-scorecard.test.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { runCheck } from "../src/index.js"; +import { buildDoctorOutputContract } from "../src/core/output-contract.js"; import { buildJsonReport } from "../src/reporting/render-json-report.js"; describe("runtime scorecard", () => { @@ -95,4 +96,83 @@ describe("runtime scorecard", () => { overall: "warn" }); }); + + it("adds a complete remote reliability scorecard without changing the prior remote shape", () => { + const reliability = { + getSse: "pass" as const, + sessionPropagation: "pass" as const, + resumability: "skipped" as const, + disconnectSafety: "skipped" as const, + sessionRestart: "skipped" as const, + termination: "skipped" as const, + overall: "pass" as const + }; + const report = buildJsonReport( + { + targetPath: "/test/plugin", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + initialize: "skipped", + toolsList: "unsupported", + toolsCall: "unsupported", + resourcesList: "unsupported", + resourceRead: "unsupported", + resourceTemplatesList: "unsupported", + promptsList: "unsupported", + promptGet: "unsupported", + remote: { + transport: "pass", + networkSafety: "pass", + initialize: "pass", + contentType: "pass", + session: "absent", + protocolHeaders: "pass", + authorization: "skipped", + reliability, + overall: "pass" + } + } + }, + { runtimeProbeEnabled: true } + ); + + expect(report.summary.runtimeScorecard?.remote?.reliability).toEqual(reliability); + }); + + it("keeps reliability optional while rejecting unsupported capability statuses", () => { + const contract = buildDoctorOutputContract("2026-07-26T00:00:00.000Z"); + const schema = contract.schemas.find((entry) => entry.id === "doctor.check.json")?.schema; + const remote = (schema?.properties as { summary?: { properties?: { runtimeScorecard?: { properties?: { remote?: { properties?: Record; required?: string[] } } } } } }) + .summary?.properties?.runtimeScorecard?.properties?.remote; + const reliability = remote?.properties?.reliability as { + properties?: Record; + required?: string[]; + additionalProperties?: boolean; + } | undefined; + + expect(remote?.required).not.toContain("reliability"); + expect(reliability).toMatchObject({ + required: ["getSse", "sessionPropagation", "resumability", "disconnectSafety", "sessionRestart", "termination", "overall"], + additionalProperties: false + }); + for (const capability of [ + "getSse", + "sessionPropagation", + "resumability", + "disconnectSafety", + "sessionRestart", + "termination", + "overall" + ]) { + expect(reliability?.properties?.[capability]?.enum).not.toContain("unsupported"); + expect(reliability?.properties?.[capability]?.enum).toEqual([ + "pass", + "warn", + "fail", + "skipped" + ]); + } + }); }); diff --git a/tests/markdown-report.test.ts b/tests/markdown-report.test.ts index 0aa1fff..86d7f65 100644 --- a/tests/markdown-report.test.ts +++ b/tests/markdown-report.test.ts @@ -100,6 +100,27 @@ describe("buildMarkdownReport", () => { expect(report).toContain("| Session | PRESENT-VALID |"); }); + it("renders remote reliability capability labels and statuses only", () => { + const report = buildMarkdownReport({ + targetPath: "example", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + ...runtimeScorecard, + remote: { + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "present-valid", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "warn", sessionRestart: "skipped", termination: "skipped", overall: "warn" } + } + } + }, { runtimeProbeEnabled: true }); + + expect(report).toContain("## Remote Transport Reliability"); + expect(report).toContain("| GET SSE | PASS |"); + expect(report).toContain("| Disconnect safety | WARN |"); + expect(report).not.toMatch(/session-secret-sentinel|event-secret-sentinel|retry-secret-sentinel|body-secret-sentinel/); + }); + it("renders a CI-friendly markdown summary", async () => { const targetPath = path.resolve("tests/fixtures/heuristic-long-plugin-description"); const result = await runCheck(targetPath); diff --git a/tests/mcp-command.test.ts b/tests/mcp-command.test.ts index dc6bc24..7bd8034 100644 --- a/tests/mcp-command.test.ts +++ b/tests/mcp-command.test.ts @@ -36,7 +36,7 @@ async function createStandaloneMcpPackage(mcpConfig: unknown): Promise { return targetPath; } -async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = {}): Promise<{ +async function startRemoteMcpServer(options: { invalidInitialize?: boolean; session?: boolean; incompleteSse?: boolean } = {}): Promise<{ url: string; requests: string[]; close(): Promise; @@ -46,6 +46,24 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { const chunks: Buffer[] = []; request.on("data", (chunk: Buffer) => chunks.push(chunk)); request.on("end", () => { + if (request.method === "GET") { + requests.push("GET"); + if (options.incompleteSse) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("id: bounded-event\\n"); + return; + } + response.writeHead(405); + response.end(); + return; + } + + if (request.method === "DELETE") { + requests.push("DELETE"); + response.writeHead(204); + response.end(); + return; + } const message = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { method: string }; requests.push(message.method); @@ -56,7 +74,10 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { return; } - response.writeHead(200, { "content-type": "application/json" }); + response.writeHead(200, { + "content-type": "application/json", + ...(options.session ? { "mcp-session-id": "session-for-test" } : {}) + }); response.end(JSON.stringify({ jsonrpc: "2.0", id: 1, @@ -69,7 +90,7 @@ async function startRemoteMcpServer(options: { invalidInitialize?: boolean } = { return; } - response.writeHead(204); + response.writeHead(202); response.end(); }); }); @@ -118,7 +139,7 @@ describe("mcp command", () => { } } }); - expect(remote.requests).toEqual(["initialize", "notifications/initialized"]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await remote.close(); } @@ -141,12 +162,76 @@ describe("mcp command", () => { expect(JSON.parse(stdout.join(""))).toMatchObject({ runtimeScorecard: { remote: { networkSafety: "pass", overall: "pass" } } }); - expect(remote.requests).toEqual(["initialize", "notifications/initialized"]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await remote.close(); } }); + it("requires lifecycle consent before terminating a remote session", async () => { + const remote = await startRemoteMcpServer({ session: true }); + const targetPath = await createStandaloneMcpPackage({ mcpServers: { local: { url: remote.url } } }); + const { io, stderr } = createIo(); + + try { + const exitCode = await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--allow-session-lifecycle", "--json" + ], io); + + expect(exitCode).toBe(0); + expect(stderr).toEqual([]); + expect(remote.requests).toEqual(["initialize", "notifications/initialized", "GET", "DELETE"]); + } finally { + await remote.close(); + } + }); + + it("turns an inconclusive remote reliability scorecard into a blocking result only when requested", async () => { + const remote = await startRemoteMcpServer({ incompleteSse: true }); + const targetPath = await createStandaloneMcpPackage({ mcpServers: { local: { url: remote.url } } }); + const withoutGate = createIo(); + const withGate = createIo(); + + try { + expect(await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--json" + ], withoutGate.io)).toBe(0); + expect(await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--require-remote-reliability", "--json" + ], withGate.io)).toBe(1); + } finally { + await remote.close(); + } + }); + + it("blocks the strict gate when one of multiple remote reliability scorecards warns", async () => { + const passing = await startRemoteMcpServer(); + const warning = await startRemoteMcpServer({ incompleteSse: true }); + const targetPath = await createStandaloneMcpPackage({ + mcpServers: { + passing: { url: passing.url }, + warning: { url: warning.url } + } + }); + const { io, stdout, stderr } = createIo(); + + try { + const exitCode = await runCli([ + "mcp", targetPath, "--runtime", "--allow-network", "--allow-local-network", "--require-remote-reliability", "--json" + ], io); + const output = JSON.parse(stdout.join("")); + + expect(exitCode).toBe(1); + expect(stderr).toEqual([]); + expect(output.runtimeScorecard.remote.reliability.overall).toBe("warn"); + expect(passing.requests).toEqual(["initialize", "notifications/initialized", "GET"]); + expect(warning.requests).toEqual(["initialize", "notifications/initialized", "GET"]); + } finally { + await passing.close(); + await warning.close(); + } + }); + it("preserves the worst remote status when a later remote server passes", async () => { const failing = await startRemoteMcpServer({ invalidInitialize: true }); const passing = await startRemoteMcpServer(); @@ -171,7 +256,7 @@ describe("mcp command", () => { overall: "fail" }); expect(failing.requests).toEqual(["initialize"]); - expect(passing.requests).toEqual(["initialize", "notifications/initialized"]); + expect(passing.requests).toEqual(["initialize", "notifications/initialized", "GET"]); } finally { await failing.close(); await passing.close(); diff --git a/tests/public-readiness.test.ts b/tests/public-readiness.test.ts index dee991b..d2e4c51 100644 --- a/tests/public-readiness.test.ts +++ b/tests/public-readiness.test.ts @@ -101,6 +101,9 @@ describe("public repository readiness", () => { const actionGuide = await readText("docs/guides/github-action.md"); const conformance = await readText("docs/architecture/mcp-2025-11-conformance.md"); const readiness = await readText("docs/architecture/remote-mcp-readiness.md"); + const reliability = await readText("docs/architecture/remote-mcp-transport-reliability.md"); + const releaseGating = await readText("docs/guides/release-gating.md"); + const runtimeSecurity = await readText("docs/security/runtime-approval-and-sandboxing.md"); const docsReadme = await readText("docs/README.md"); const security = await readText("docs/security/security-architecture.md"); expect(readme).toContain("Remote MCP Readiness"); @@ -113,9 +116,26 @@ describe("public repository readiness", () => { expect(readiness).toContain("authenticated OAuth"); expect(readiness).toContain("custom headers"); expect(readiness).toContain("remote tool/resource/prompt/task calls"); - expect(readiness).toContain("GET SSE/resumability"); + expect(readiness).toContain("one bounded GET request"); expect(readiness).toContain("redirects"); expect(docsReadme).toContain("Remote MCP Readiness"); + expect(docsReadme).toContain("Remote MCP Transport Reliability"); + expect(readme).toContain("--allow-session-lifecycle"); + expect(readme).toContain("--require-remote-reliability"); + expect(readiness).toContain("--allow-session-lifecycle"); + expect(readiness).toContain("--require-remote-reliability"); + expect(reliability).toContain("GET returning 405"); + expect(reliability).toContain("--runtime --allow-network"); + expect(reliability).toContain("one bounded `DELETE` request"); + expect(reliability).toContain("does not claim delivery guarantees"); + expect(reliability).toContain("GET carrying an issued session identifier"); + expect(reliability).toContain("DELETE 404 never restarts or retries"); + for (const document of [reliability, readme, readiness, releaseGating, runtimeSecurity, actionGuide]) { + expect(document).toMatch(/every attempted remote\s+reliability scorecard passes/); + expect(document).toContain("Local-only runs are unaffected."); + } + expect(releaseGating).toContain("--require-remote-reliability"); + expect(runtimeSecurity).toContain("--allow-session-lifecycle"); expect(security).toContain("runner or host egress controls"); expect(readiness).not.toMatch(/internal (implementation )?plan/i); }); diff --git a/tests/release-check-command.test.ts b/tests/release-check-command.test.ts index ace816b..e276187 100644 --- a/tests/release-check-command.test.ts +++ b/tests/release-check-command.test.ts @@ -88,6 +88,29 @@ describe("release-check command", () => { }); describe("CLI", () => { + it("forwards lifecycle consent and strict reliability gating to release validation", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + await runCli([ + "release", "check", "tests/fixtures/valid-plugin-with-mcp", "--runtime", "--allow-network", + "--allow-session-lifecycle", "--require-remote-reliability", "--json" + ], io, { runCheckImpl }); + + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + it("keeps runtime probing disabled by default", async () => { const { io, stdout, stderr } = createIo(); const runCheckImpl = vi.fn(async (targetPath: string) => ({ diff --git a/tests/release-evidence-command.test.ts b/tests/release-evidence-command.test.ts index 741425e..17e4a26 100644 --- a/tests/release-evidence-command.test.ts +++ b/tests/release-evidence-command.test.ts @@ -25,6 +25,33 @@ function createIo() { } describe("doctor release-evidence command", () => { + it("forwards lifecycle consent and strict reliability gating to release evidence validation", async () => { + const { io, stderr } = createIo(); + const runCheckImpl = vi.fn(async (targetPath: string) => ({ + targetPath, + status: "pass" as const, + exitCode: 0 as const, + findings: [] + })); + + await runCli([ + "doctor", "release-evidence", "examples/codex-doctor-runtime", "--sign-key-env", "DOCTOR_SIGNING_KEY", + "--allow-dirty", "--allow-untagged", "--runtime", "--allow-network", "--allow-session-lifecycle", + "--require-remote-reliability", "--json" + ], io, { + terminalContext: { env: { DOCTOR_SIGNING_KEY: "release-secret" }, platform: "win32" }, + runCheckImpl + }); + + expect(stderr).toEqual([]); + expect(runCheckImpl).toHaveBeenCalledWith(expect.any(String), { + runtime: true, + allowNetwork: true, + allowSessionLifecycle: true, + requireRemoteReliability: true + }); + }); + it("renders a signed release evidence bundle as JSON", async () => { const { io, stdout, stderr } = createIo(); @@ -430,6 +457,28 @@ describe("doctor release-evidence command", () => { expect(rendered).not.toContain("SHOULD_NOT_LEAK"); }); + it("never renders remote transport values in release evidence", () => { + const rendered = renderDoctorReleaseEvidenceJson({ + kind: "doctor.release.evidence", + schemaVersion: "1.0.0", + transport: { + sessionId: "release-session-canary-7d31", + lastEventId: "release-event-canary-91ac", + retry: "release-retry-canary-4f82", + data: "release-sse-data-canary-c8e5" + } + } as never); + + for (const canary of [ + "release-session-canary-7d31", + "release-event-canary-91ac", + "release-retry-canary-4f82", + "release-sse-data-canary-c8e5" + ]) { + expect(rendered).not.toContain(canary); + } + }); + it("verifies a release evidence bundle against its target package", async () => { const outputPath = path.join( await mkdtemp(path.join(os.tmpdir(), "codex-plugin-doctor-release-evidence-verify-")), diff --git a/tests/remote-mcp-probe.test.ts b/tests/remote-mcp-probe.test.ts index 651a5d3..bcbe809 100644 --- a/tests/remote-mcp-probe.test.ts +++ b/tests/remote-mcp-probe.test.ts @@ -1,20 +1,28 @@ import { createServer, type Server, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import os from "node:os"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { probeRemoteMcpServer } from "../src/core/remote-mcp-probe.js"; +import { probeRuntimeConfig } from "../src/core/runtime-probe.js"; import type { BoundedHttpResponse } from "../src/core/bounded-http-client.js"; import type { RemoteLookup } from "../src/core/remote-network-policy.js"; import { packageVersion } from "../src/version.js"; const servers: Server[] = []; const openResponses: ServerResponse[] = []; +const temporaryDirectories: string[] = []; afterEach(async () => { openResponses.splice(0).forEach((response) => response.destroy()); - await Promise.all(servers.splice(0).map((server) => new Promise((resolve, reject) => { - server.close((error) => error ? reject(error) : resolve()); - }))); + await Promise.all([ + ...servers.splice(0).map((server) => new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + })), + ...temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })) + ]); }); async function startServer(handler: Parameters[0]): Promise { @@ -71,6 +79,11 @@ describe("probeRemoteMcpServer", () => { request.on("data", (chunk) => { body += chunk; }); request.on("end", () => { requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } const message = JSON.parse(body) as { id?: number; method: string }; if (message.method === "initialize") { response.writeHead(200, { @@ -96,11 +109,20 @@ describe("probeRemoteMcpServer", () => { session: "present-valid", protocolHeaders: "pass", authorization: "skipped", + reliability: { + getSse: "pass", + sessionPropagation: "pass", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "pass" + }, overall: "pass" }); - expect(requests).toHaveLength(2); - expect(requests.map((request) => request.method)).toEqual(["POST", "POST"]); - expect(requests.map((request) => JSON.parse(request.body).method)).toEqual([ + expect(requests).toHaveLength(3); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET"]); + expect(requests.slice(0, 2).map((request) => JSON.parse(request.body).method)).toEqual([ "initialize", "notifications/initialized" ]); @@ -120,6 +142,276 @@ describe("probeRemoteMcpServer", () => { expect(requests[0]?.headers.cookie).toBeUndefined(); expect(requests[1]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); expect(requests[1]?.headers["mcp-session-id"]).toBe("session-secret-sentinel"); + expect(requests[2]?.headers["mcp-session-id"]).toBe("session-secret-sentinel"); + assertPrivate(result); + }); + + it.each([ + ["HTTP 200", 200, ""], + ["another 2xx status", 204, ""], + ["a non-empty HTTP 202 response", 202, "notification-body-secret-sentinel"] + ])("rejects %s for the initial initialized notification", async (_name, statusCode, responseBody) => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(statusCode); + response.end(responseBody); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.initialized.failed", severity: "fail" }) + ]); + expect(result.scorecard).toMatchObject({ protocolHeaders: "fail", overall: "fail" }); + expect(JSON.stringify(result)).not.toContain("notification-body-secret-sentinel"); + assertPrivate(result); + }); + + it("merges reliability scorecards across remote servers field by field", async () => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(request.url === "/failing" ? 500 : 405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + const rootPath = await mkdtemp(path.join(os.tmpdir(), "codex-plugin-doctor-remote-merge-")); + temporaryDirectories.push(rootPath); + await writeFile(path.join(rootPath, ".mcp.json"), JSON.stringify({ + mcpServers: { + compliant: { url: `http://localhost:${port}/compliant` }, + failing: { url: `http://localhost:${port}/failing` } + } + }), "utf8"); + + const result = await probeRuntimeConfig(rootPath, ".mcp.json", { + allowNetwork: true, + allowLocalNetwork: true, + remoteLookup: localLookup(), + remoteRequestTimeoutMs: 100 + }); + + expect(result.scorecard.remote?.reliability).toEqual({ + getSse: "fail", + sessionPropagation: "skipped", + resumability: "skipped", + disconnectSafety: "skipped", + sessionRestart: "skipped", + termination: "skipped", + overall: "fail" + }); + expect(result.findings).toEqual([ + expect.objectContaining({ + id: "plugin.runtime.remote.reliability.get.status", + severity: "fail", + evidence: expect.objectContaining({ serverName: "failing" }) + }) + ]); + }); + + it("treats a bounded SSE GET 405 as protocol-compliant after initialization", async () => { + const requests: Array<{ method: string; headers: Record; body: string }> = []; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([]); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET"]); + expect(requests[2]?.headers.accept).toBe("text/event-stream"); + expect(requests[2]?.headers["mcp-protocol-version"]).toBe("2025-11-25"); + }); + + it("publishes redacted reliability failures through the remote scorecard and findings", async () => { + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + if (request.method === "GET") { + response.writeHead(500, { "content-type": "application/json" }); + response.end('{"error":"remote-body-secret-sentinel"}'); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.get.status", severity: "fail" }) + ]); + expect(result.scorecard.overall).toBe("fail"); + expect(result.scorecard.reliability?.overall).toBe("fail"); + assertPrivate(result); + }); + + it("shares one fixed request budget with both actual session-restart POSTs", async () => { + const requests: Array<{ method: string; headers: Record; body: string }> = []; + let initializeCount = 0; + let getCount = 0; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", headers: request.headers, body }); + if (request.method === "GET") { + getCount += 1; + if (getCount === 1) { + response.writeHead(404); + } else if (getCount === 2) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("id: event-secret-sentinel\n\n"); + return; + } else if (getCount === 3) { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("\n"); + return; + } else { + response.writeHead(500); + } + response.end(); + return; + } + if (request.method === "DELETE") { + response.writeHead(204); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + initializeCount += 1; + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": initializeCount === 1 + ? "session-secret-sentinel" + : "replacement-session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(202); + response.end(); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, { + ...options(port), + allowSessionLifecycle: true + }); + + expect(result.findings).toEqual([]); + expect(result.scorecard.reliability).toMatchObject({ sessionRestart: "pass", resumability: "pass", termination: "pass", overall: "pass" }); + expect(requests.map((request) => request.method)).toEqual([ + "POST", "POST", "GET", "POST", "POST", "GET", "GET", "DELETE" + ]); + expect(requests).toHaveLength(8); + expect(initializeCount).toBe(2); + assertPrivate(result); + }); + + it.each([ + ["HTTP 200", 200, ""], + ["another 2xx status", 204, ""], + ["a non-empty HTTP 202 response", 202, "notification-body-secret-sentinel"] + ])("rejects %s for the replacement session initialized notification", async (_name, statusCode, responseBody) => { + let initializeCount = 0; + const requests: Array<{ method: string; body: string }> = []; + const port = await startServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + requests.push({ method: request.method ?? "", body }); + if (request.method === "GET") { + response.writeHead(404); + response.end(); + return; + } + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.method === "initialize") { + initializeCount += 1; + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": initializeCount === 1 + ? "session-secret-sentinel" + : "replacement-session-secret-sentinel" + }); + response.end(initializedResponse(message.id ?? 1)); + return; + } + response.writeHead(initializeCount === 1 ? 202 : statusCode); + response.end(initializeCount === 1 ? "" : responseBody); + }); + }); + + const result = await probeRemoteMcpServer("remote", options(port).url, options(port)); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" }) + ]); + expect(result.scorecard.reliability).toMatchObject({ sessionRestart: "fail", overall: "fail" }); + expect(requests.map((request) => request.method)).toEqual(["POST", "POST", "GET", "POST", "POST"]); + expect(JSON.stringify(result)).not.toContain("notification-body-secret-sentinel"); assertPrivate(result); }); @@ -129,6 +421,11 @@ describe("probeRemoteMcpServer", () => { request.setEncoding("utf8"); request.on("data", (chunk) => { body += chunk; }); request.on("end", () => { + if (request.method === "GET") { + response.writeHead(405); + response.end(); + return; + } const message = JSON.parse(body) as { id?: number; method: string }; if (message.method === "initialize") { response.writeHead(200, { "content-type": "text/event-stream" }); diff --git a/tests/remote-transport-reliability.test.ts b/tests/remote-transport-reliability.test.ts new file mode 100644 index 0000000..3d728e8 --- /dev/null +++ b/tests/remote-transport-reliability.test.ts @@ -0,0 +1,416 @@ +import { describe, expect, it } from "vitest"; + +import { BoundedHttpError, type BoundedHttpRequestOptions, type BoundedHttpResponse } from "../src/core/bounded-http-client.js"; +import { probeRemoteTransportReliability } from "../src/core/remote-transport-reliability.js"; +import { renderJsonReport } from "../src/reporting/render-json-report.js"; +import { buildMarkdownReport } from "../src/reporting/render-markdown-report.js"; +import { renderTextReport } from "../src/reporting/render-text-report.js"; + +const endpoint = "https://mcp.example/mcp"; +const protocolVersion = "2025-11-25"; + +function response(statusCode: number, contentType?: string, body = ""): BoundedHttpResponse { + return { + statusCode, + headers: contentType === undefined ? {} : { "content-type": contentType }, + body: Buffer.from(body) + }; +} + +function scriptedRequest(responses: Array) { + const requests: BoundedHttpRequestOptions[] = []; + return { + requests, + request: async (_url: string, options?: BoundedHttpRequestOptions): Promise => { + requests.push(options ?? {}); + const next = responses.shift(); + if (next instanceof Error) throw next; + if (next === undefined) throw new Error("unexpected request"); + return next; + } + }; +} + +function probe(request: (url: string, options?: BoundedHttpRequestOptions) => Promise, overrides: { + sessionId?: string | null; + allowSessionLifecycle?: boolean; + requestTimeoutMs?: number; + reinitialize?: () => Promise; +} = {}) { + return probeRemoteTransportReliability({ + rawUrl: endpoint, + protocolVersion, + request, + requestTimeoutMs: overrides.requestTimeoutMs ?? 50, + sessionId: overrides.sessionId ?? null, + allowSessionLifecycle: overrides.allowSessionLifecycle, + reinitialize: overrides.reinitialize ?? (async () => null) + }); +} + +function assertRedacted(value: unknown): void { + const serialized = JSON.stringify(value); + for (const secret of [ + "session-secret-sentinel", + "event-secret-sentinel", + "999999", + "remote-body-secret-sentinel" + ]) { + expect(serialized).not.toContain(secret); + } +} + +describe("probeRemoteTransportReliability", () => { + it("keeps remote transport canaries out of CLI and Action-facing report content", async () => { + const fixture = scriptedRequest([ + response( + 200, + "text/event-stream", + "id: report-event-canary-91ac\nretry: 917431\ndata: report-sse-data-canary-c8e5\n\n" + ) + ]); + const result = await probe(fixture.request, { + sessionId: "report-session-canary-7d31", + requestTimeoutMs: 10 + }); + const report = { + targetPath: "example", + status: "warn" as const, + exitCode: 0, + findings: result.findings, + runtimeScorecard: { + initialize: "pass" as const, + toolsList: "skipped" as const, + toolsCall: "skipped" as const, + resourcesList: "skipped" as const, + resourceRead: "skipped" as const, + resourceTemplatesList: "skipped" as const, + promptsList: "skipped" as const, + promptGet: "skipped" as const, + remote: { + transport: "pass" as const, + networkSafety: "pass" as const, + initialize: "pass" as const, + contentType: "pass" as const, + session: "present-valid" as const, + protocolHeaders: "pass" as const, + authorization: "skipped" as const, + overall: "warn" as const, + reliability: result.scorecard + } + } + }; + + for (const output of [ + renderJsonReport(report, { runtimeProbeEnabled: true }), + renderTextReport(report), + buildMarkdownReport(report, { runtimeProbeEnabled: true }) + ]) { + for (const canary of [ + "report-session-canary-7d31", + "report-event-canary-91ac", + "917431", + "report-sse-data-canary-c8e5" + ]) { + expect(output).not.toContain(canary); + } + } + }); + + it("accepts GET 405 as compliant without attempting resume", async () => { + const fixture = scriptedRequest([response(405)]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", resumability: "skipped", overall: "pass" }); + expect(fixture.requests).toHaveLength(1); + }); + + it("uses a safe event id for exactly one resume without retaining it", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\ndata: remote-body-secret-sentinel\n\n"), + response(200, "text/event-stream", "\n"), + response(204) + ]); + + const result = await probe(fixture.request, { sessionId: "session-secret-sentinel", allowSessionLifecycle: true }); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", sessionPropagation: "pass", resumability: "pass", overall: "pass" }); + expect(fixture.requests).toHaveLength(3); + expect(fixture.requests[0]?.headers).toMatchObject({ + Accept: "text/event-stream", + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel" + }); + expect(fixture.requests[0]?.headers?.["Last-Event-ID"]).toBeUndefined(); + expect(fixture.requests[1]?.headers).toMatchObject({ + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel", + "Last-Event-ID": "event-secret-sentinel" + }); + expect(fixture.requests[2]?.headers).toMatchObject({ + "MCP-Protocol-Version": protocolVersion, + "MCP-Session-Id": "session-secret-sentinel" + }); + expect(fixture.requests[2]?.headers?.["Last-Event-ID"]).toBeUndefined(); + assertRedacted(result); + }); + + it("accepts a complete SSE event without an id without attempting resume", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "event: message\ndata: remote-body-secret-sentinel\n\n") + ]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ getSse: "pass", resumability: "skipped", overall: "pass" }); + expect(fixture.requests).toHaveLength(1); + assertRedacted(result); + }); + + it("honors a retry delay only when it remains within the reliability deadline", async () => { + const withinDeadline = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\nretry: 1\n\n"), + response(200, "text/event-stream", "\n") + ]); + const withinResult = await probe(withinDeadline.request, { requestTimeoutMs: 50 }); + + const overDeadline = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\nretry: 999999\n\n") + ]); + const overResult = await probe(overDeadline.request, { requestTimeoutMs: 10 }); + + expect(withinResult.scorecard.resumability).toBe("pass"); + expect(withinDeadline.requests).toHaveLength(2); + expect(overResult.scorecard.resumability).toBe("warn"); + expect(overDeadline.requests).toHaveLength(1); + assertRedacted(overResult); + }); + + it.each([ + [response(500), "plugin.runtime.remote.reliability.get.status"], + [response(200, "application/json"), "plugin.runtime.remote.reliability.get.content_type"], + [response(200, "text/event-stream", "id: invalid event-secret-sentinel\n\n"), "plugin.runtime.remote.reliability.get.malformed"] + ])("fails invalid GET evidence with stable redacted findings", async (fixtureResponse, findingId) => { + const fixture = scriptedRequest([fixtureResponse]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([expect.objectContaining({ id: findingId, severity: "fail" })]); + expect(result.scorecard.overall).toBe("fail"); + assertRedacted(result); + }); + + it("treats an incomplete SSE response after headers as inconclusive", async () => { + const timeout = new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "timeout", 200, { "content-type": "text/event-stream" }); + const fixture = scriptedRequest([timeout]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.get.inconclusive", severity: "warn" })]); + expect(result.scorecard.overall).toBe("warn"); + }); + + it("treats a bounded SSE resume timeout as inconclusive", async () => { + const timeout = new BoundedHttpError("REMOTE_HTTP_TIMEOUT", "timeout", 200, { "content-type": "text/event-stream" }); + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\n\n"), + timeout + ]); + + const result = await probe(fixture.request); + + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.resume.inconclusive", severity: "warn" }) + ]); + expect(result.scorecard).toMatchObject({ resumability: "warn", overall: "warn" }); + assertRedacted(result); + }); + + it("reinitializes once after a session-bound GET 404 and uses the replacement session", async () => { + const fixture = scriptedRequest([response(404), response(405)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(2); + expect(fixture.requests[1]?.headers?.["MCP-Session-Id"]).toBe("replacement-session-secret-sentinel"); + expect(result.scorecard).toMatchObject({ sessionRestart: "pass", overall: "pass" }); + assertRedacted(result); + }); + + it("does not carry a resume cursor into a replacement MCP session", async () => { + const fixture = scriptedRequest([ + response(200, "text/event-stream", "id: event-secret-sentinel\n\n"), + response(404), + response(405) + ]); + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => "replacement-session-secret-sentinel" + }); + + expect(fixture.requests).toHaveLength(3); + expect(fixture.requests[1]?.headers?.["Last-Event-ID"]).toBe("event-secret-sentinel"); + expect(fixture.requests[2]?.headers).toMatchObject({ + "MCP-Session-Id": "replacement-session-secret-sentinel" + }); + expect(fixture.requests[2]?.headers?.["Last-Event-ID"]).toBeUndefined(); + expect(result.findings).toEqual([]); + expect(result.scorecard).toMatchObject({ + sessionRestart: "pass", + resumability: "skipped", + overall: "pass" + }); + assertRedacted(result); + }); + + it("counts both restart POST requests against the shared reliability request budget", async () => { + const fixture = scriptedRequest([ + response(404), + response(200, "application/json", "{}"), + response(202), + response(405) + ]); + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async (requestWithinBudget: unknown) => { + if (typeof requestWithinBudget !== "function") { + throw new Error("restart requests must share the reliability budget"); + } + const bounded = requestWithinBudget as (options: BoundedHttpRequestOptions) => Promise; + await bounded({ method: "POST", headers: { "Content-Type": "application/json" } }); + await bounded({ method: "POST", headers: { "Content-Type": "application/json" } }); + return "replacement-session-secret-sentinel"; + } + }); + + expect(result.scorecard).toMatchObject({ sessionRestart: "pass", overall: "pass" }); + expect(fixture.requests.map((request) => request.method)).toEqual(["GET", "POST", "POST", "GET"]); + expect(fixture.requests).toHaveLength(4); + assertRedacted(result); + }); + + it("refuses a seventh request through the restart budget", async () => { + const fixture = scriptedRequest([ + response(404), + response(202), + response(202), + response(202), + response(202), + response(202) + ]); + let ceilingReached = false; + + await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async (requestWithinBudget: unknown) => { + if (typeof requestWithinBudget !== "function") { + throw new Error("restart requests must share the reliability budget"); + } + const bounded = requestWithinBudget as (options: BoundedHttpRequestOptions) => Promise; + for (let index = 0; index < 5; index += 1) { + await bounded({ method: "POST" }); + } + try { + await bounded({ method: "POST" }); + } catch { + ceilingReached = true; + } + return "replacement-session-secret-sentinel"; + } + }); + + expect(ceilingReached).toBe(true); + expect(fixture.requests).toHaveLength(6); + }); + + it("fails a single bounded session restart without recursion", async () => { + const fixture = scriptedRequest([response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + throw new Error("session-secret-sentinel"); + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(1); + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" })]); + assertRedacted(result); + }); + + it("does not restart a replacement session or exceed the request ceiling", async () => { + const fixture = scriptedRequest([response(404), response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(restarts).toBe(1); + expect(fixture.requests).toHaveLength(2); + expect(result.scorecard.sessionRestart).toBe("fail"); + expect(result.findings).toEqual([expect.objectContaining({ id: "plugin.runtime.remote.reliability.session_restart.failed", severity: "fail" })]); + assertRedacted(result); + }); + + it("treats a session-bound DELETE 404 as one failed termination without restart", async () => { + const fixture = scriptedRequest([response(405), response(404)]); + let restarts = 0; + + const result = await probe(fixture.request, { + sessionId: "session-secret-sentinel", + allowSessionLifecycle: true, + reinitialize: async () => { + restarts += 1; + return "replacement-session-secret-sentinel"; + } + }); + + expect(fixture.requests.map((request) => request.method)).toEqual(["GET", "DELETE"]); + expect(restarts).toBe(0); + expect(result.scorecard).toMatchObject({ termination: "fail", overall: "fail" }); + expect(result.findings).toEqual([ + expect.objectContaining({ id: "plugin.runtime.remote.reliability.termination.failed", severity: "fail" }) + ]); + assertRedacted(result); + }); + + it.each([ + [false, response(405), "skipped", "pass"], + [true, response(204), "pass", "pass"], + [true, response(405), "pass", "pass"], + [true, response(500), "fail", "fail"] + ])("limits DELETE lifecycle behavior to explicit consent", async (allowSessionLifecycle, deleteResponse, termination, overall) => { + const fixture = scriptedRequest([response(405), deleteResponse]); + + const result = await probe(fixture.request, { sessionId: "session-secret-sentinel", allowSessionLifecycle }); + + expect(result.scorecard.termination).toBe(termination); + expect(result.scorecard.overall).toBe(overall); + expect(fixture.requests.map((request) => request.method)).toEqual( + allowSessionLifecycle ? ["GET", "DELETE"] : ["GET"] + ); + assertRedacted(result); + }); +}); diff --git a/tests/render-text-report.test.ts b/tests/render-text-report.test.ts index b191bcb..a3c6e65 100644 --- a/tests/render-text-report.test.ts +++ b/tests/render-text-report.test.ts @@ -93,6 +93,27 @@ describe("renderTextReport", () => { expect(output).toContain("session: present-valid"); }); + it("renders remote reliability capability labels and statuses only", () => { + const output = renderTextReport({ + targetPath: "example", + status: "pass", + exitCode: 0, + findings: [], + runtimeScorecard: { + ...runtimeScorecard, + remote: { + transport: "pass", networkSafety: "pass", initialize: "pass", contentType: "pass", session: "present-valid", protocolHeaders: "pass", authorization: "skipped", overall: "pass", + reliability: { getSse: "pass", sessionPropagation: "pass", resumability: "skipped", disconnectSafety: "warn", sessionRestart: "skipped", termination: "skipped", overall: "warn" } + } + } + }); + + expect(output).toContain("Remote Transport Reliability"); + expect(output).toContain("GET SSE: pass"); + expect(output).toContain("Disconnect safety: warn"); + expect(output).not.toMatch(/session-secret-sentinel|event-secret-sentinel|retry-secret-sentinel|body-secret-sentinel/); + }); + it("renders a rich unicode summary for warn results", async () => { const result = await runCheck( path.resolve("tests/fixtures/heuristic-long-plugin-description") diff --git a/tests/rule-catalog.test.ts b/tests/rule-catalog.test.ts index 575bdbe..fcc467d 100644 --- a/tests/rule-catalog.test.ts +++ b/tests/rule-catalog.test.ts @@ -104,6 +104,24 @@ const remoteRuntimeRules = [ "plugin.runtime.remote.authorization.metadata.unavailable" ] as const; +const remoteReliabilityFailRules = [ + "plugin.runtime.remote.reliability.get.status", + "plugin.runtime.remote.reliability.get.content_type", + "plugin.runtime.remote.reliability.get.failed", + "plugin.runtime.remote.reliability.get.malformed", + "plugin.runtime.remote.reliability.resume.status", + "plugin.runtime.remote.reliability.resume.content_type", + "plugin.runtime.remote.reliability.resume.failed", + "plugin.runtime.remote.reliability.resume.malformed", + "plugin.runtime.remote.reliability.session_restart.failed", + "plugin.runtime.remote.reliability.termination.failed" +] as const; + +const remoteReliabilityWarnRules = [ + "plugin.runtime.remote.reliability.get.inconclusive", + "plugin.runtime.remote.reliability.resume.inconclusive" +] as const; + describe("MCP 2025-11 conformance rule catalog", () => { it("resolves every evaluator finding with its public remediation contract", () => { expect(ruleCatalog.filter((rule) => rule.id.startsWith("mcp.conformance."))).toEqual( @@ -126,4 +144,16 @@ describe("MCP 2025-11 conformance rule catalog", () => { expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "fail" }); } }); + + it("resolves failing remote reliability findings with fail remediation contracts", () => { + for (const id of remoteReliabilityFailRules) { + expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "fail" }); + } + }); + + it("resolves inconclusive remote reliability findings with warn remediation contracts", () => { + for (const id of remoteReliabilityWarnRules) { + expect(findRuleDefinition(id)).toMatchObject({ id, category: "runtime", defaultSeverity: "warn" }); + } + }); }); diff --git a/tests/sse-observation.test.ts b/tests/sse-observation.test.ts new file mode 100644 index 0000000..de5d034 --- /dev/null +++ b/tests/sse-observation.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; + +import { observeFirstSseEvent } from "../src/core/sse-observation.js"; + +describe("observeFirstSseEvent", () => { + it("observes the first complete CRLF-delimited event without returning its data", () => { + const observation = observeFirstSseEvent(Buffer.from( + ": ready\r\nid: event-1\r\nretry: 2500\r\ndata: secret payload\r\nunknown: ignored\r\n\r\nid: event-2\r\n\r\n" + )); + + expect(observation).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 2500, + malformed: false + }); + expect(observation).not.toHaveProperty("data"); + }); + + it("does not observe an incomplete event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\ndata: secret payload\n"))).toEqual({ + complete: false, + eventId: null, + retryMs: null, + malformed: false + }); + }); + + it("observes a complete LF-delimited event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\nretry: 125\n\n"))).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 125, + malformed: false + }); + }); + + it("observes only the first complete CR-delimited event", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\rretry: 125\r\rid: event-2\r\r"))).toEqual({ + complete: true, + eventId: "event-1", + retryMs: 125, + malformed: false + }); + }); + + it("does not replace a valid retry with a later invalid retry", () => { + expect(observeFirstSseEvent(Buffer.from("retry: 125\nretry: nope\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: 125, + malformed: false + }); + }); + + it.each([ + ["an overflowing retry", "9007199254740992"], + ["an oversized retry", "9".repeat(1_025)] + ])("ignores %s without replacing a valid retry", (_name, invalidRetry) => { + expect(observeFirstSseEvent(Buffer.from(`retry: 125\nretry: ${invalidRetry}\n\n`))).toEqual({ + complete: true, + eventId: null, + retryMs: 125, + malformed: false + }); + }); + + it("resets the event ID when the event contains an empty id field", () => { + expect(observeFirstSseEvent(Buffer.from("id: event-1\nid:\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: false + }); + }); + + it("rejects unsafe event IDs", () => { + expect(observeFirstSseEvent(Buffer.from("id: event\u0001-1\n\n"))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: true + }); + }); + + it("rejects oversized event ID fields", () => { + expect(observeFirstSseEvent(Buffer.from(`id: ${"a".repeat(1_025)}\n\n`))).toEqual({ + complete: true, + eventId: null, + retryMs: null, + malformed: true + }); + }); +});