feat: add conformance test infrastructure for v1.x (#1518)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>

Author: 3 people

.github/workflows/conformance.yml

@@ -0,0 +1,40 @@
+name: Conformance Tests
+
+on:
+    push:
+        branches: [v1.x]
+    pull_request:
+        branches: [v1.x]
+    workflow_dispatch:
+
+concurrency:
+    group: conformance-${{ github.ref }}
+    cancel-in-progress: true
+
+permissions:
+    contents: read
+
+jobs:
+    client-conformance:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: 20
+                  cache: npm
+            - run: npm ci
+            - run: npm run build
+            - run: npm run test:conformance:client:all
+
+    server-conformance:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: 20
+                  cache: npm
+            - run: npm ci
+            - run: npm run build
+            - run: npm run test:conformance:server

.gitignore

@@ -133,3 +133,4 @@ dist/
 
 # IDE
 .idea/
+test/conformance/node_modules/

package-lock.json

@@ -83,7 +83,12 @@
         "test:watch": "vitest",
         "start": "npm run server",
         "server": "tsx watch --clear-screen=false scripts/cli.ts server",
-        "client": "tsx scripts/cli.ts client"
+        "client": "tsx scripts/cli.ts client",
+        "test:conformance:server": "test/conformance/scripts/run-server-conformance.sh --expected-failures test/conformance/conformance-baseline.yml",
+        "test:conformance:server:all": "test/conformance/scripts/run-server-conformance.sh --suite all --expected-failures test/conformance/conformance-baseline.yml",
+        "test:conformance:server:run": "npx tsx test/conformance/src/everythingServer.ts",
+        "test:conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx test/conformance/src/everythingClient.ts' --expected-failures test/conformance/conformance-baseline.yml",
+        "test:conformance:client:all": "npx @modelcontextprotocol/conformance client --command 'npx tsx test/conformance/src/everythingClient.ts' --suite all --expected-failures test/conformance/conformance-baseline.yml"
     },
     "dependencies": {
         "@hono/node-server": "^1.19.9",
@@ -118,6 +123,7 @@
     },
     "devDependencies": {
         "@cfworker/json-schema": "^4.1.1",
+        "@modelcontextprotocol/conformance": "^0.1.11",
         "@eslint/js": "^9.39.1",
         "@types/content-type": "^1.1.8",
         "@types/cors": "^2.8.17",

package.json

@@ -0,0 +1,14 @@
+# Known conformance test failures for v1.x
+# These are tracked and should be removed as they're fixed.
+#
+# tools_call: conformance runner's test server reuses a single Server
+# instance across requests, triggering v1.26.0's "Already connected"
+# guard (GHSA-345p-7cg4-v4c7). Fixed in conformance repo (PR #141),
+# remove this entry once a new conformance release is published.
+#
+# auth/pre-registration: scenario added in conformance 0.1.11 that
+# requires a dedicated client handler for pre-registered credentials.
+# Needs to be implemented in both v1.x and main.
+client:
+    - tools_call
+    - auth/pre-registration

test/conformance/conformance-baseline.yml

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Script to run server conformance tests
+# Starts the conformance server, runs conformance tests, then stops the server
+
+set -e
+
+PORT="${PORT:-3000}"
+SERVER_URL="http://localhost:${PORT}/mcp"
+
+# Navigate to repo root
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/../../.."
+
+# Start the server in the background
+echo "Starting conformance test server on port ${PORT}..."
+npx tsx test/conformance/src/everythingServer.ts &
+SERVER_PID=$!
+
+# Function to cleanup on exit
+cleanup() {
+    echo "Stopping server (PID: ${SERVER_PID})..."
+    kill $SERVER_PID 2>/dev/null || true
+    wait $SERVER_PID 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Wait for server to be ready
+echo "Waiting for server to be ready..."
+MAX_RETRIES=30
+RETRY_COUNT=0
+while ! curl -s "${SERVER_URL}" > /dev/null 2>&1; do
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
+        echo "Server failed to start after ${MAX_RETRIES} attempts"
+        exit 1
+    fi
+    sleep 0.5
+done
+
+echo "Server is ready. Running conformance tests..."
+
+# Run conformance tests - pass through all arguments
+npx @modelcontextprotocol/conformance server --url "${SERVER_URL}" "$@"
+
+echo "Conformance tests completed."