From b5d8199f747840ea4638603483b18bb52d5baf79 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 18 May 2026 22:36:31 -0700 Subject: [PATCH 1/5] feat: add context-based fixture routing via X-AIMock-Context header Fixtures with match.context only match requests carrying that exact context value. Fixtures without context remain shared (match all). Recorder auto-captures context and routes recordings into context subdirectories. --- src/fixture-loader.ts | 11 ++++++++++- src/helpers.ts | 10 ++++++++++ src/recorder.ts | 16 +++++++++++----- src/router.ts | 7 +++++++ src/types.ts | 4 ++++ 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index adedd8af..4f80b06e 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -67,6 +67,7 @@ export function entryToFixture(entry: FixtureFileEntry, logger?: Logger): Fixtur ...(entry.match.hasToolResult !== undefined && { hasToolResult: entry.match.hasToolResult, }), + ...(entry.match.context !== undefined && { context: entry.match.context }), }, response: normalizeResponse(entry.response), ...(entry.latency !== undefined && { latency: entry.latency }), @@ -682,6 +683,13 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { }); } } + if (f.match.context !== undefined && typeof f.match.context !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `match.context must be a string, got ${typeof f.match.context}`, + }); + } // --- Warning checks --- @@ -690,7 +698,7 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { // but differ on those fields are NOT considered duplicates. const um = f.match.userMessage; if (typeof um === "string" && um) { - const dedupKey = `${um}|${f.match.turnIndex}|${f.match.hasToolResult}|${f.match.sequenceIndex}`; + const dedupKey = `${um}|${f.match.turnIndex}|${f.match.hasToolResult}|${f.match.sequenceIndex}|${f.match.context}`; const prev = seenUserMessages.get(dedupKey); if (prev !== undefined) { results.push({ @@ -707,6 +715,7 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { const match = f.match; const hasDiscriminator = match.endpoint !== undefined || + match.context !== undefined || match.userMessage !== undefined || match.systemMessage !== undefined || match.inputText !== undefined || diff --git a/src/helpers.ts b/src/helpers.ts index 1ba19128..d444b6a3 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -861,6 +861,16 @@ export function getTestId(req: http.IncomingMessage): string { return DEFAULT_TEST_ID; } +export function getContext(req: http.IncomingMessage): string | undefined { + const headerValue = req.headers["x-aimock-context"]; + if (Array.isArray(headerValue)) { + if (headerValue.length > 0 && headerValue[0]) return headerValue[0]; + } else if (typeof headerValue === "string" && headerValue) { + return headerValue; + } + return undefined; +} + // ─── Snapshot recording helpers ────────────────────────────────────────────── /** diff --git a/src/recorder.ts b/src/recorder.ts index 990accd5..8b2f2e8a 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -163,10 +163,10 @@ export function persistFixture(opts: { } } else { const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - filepath = path.join( - fixturePath, - `${providerKey}-${timestamp}-${crypto.randomUUID().slice(0, 8)}.json`, - ); + const timestampFile = `${providerKey}-${timestamp}-${crypto.randomUUID().slice(0, 8)}.json`; + filepath = fixture.match.context + ? path.join(fixturePath, fixture.match.context, timestampFile) + : path.join(fixturePath, timestampFile); } const fileWarnings = [ @@ -175,7 +175,7 @@ export function persistFixture(opts: { ]; try { - fs.mkdirSync(isSnapshotMode ? path.dirname(filepath) : fixturePath, { recursive: true }); + fs.mkdirSync(path.dirname(filepath), { recursive: true }); // Auth headers are forwarded to upstream but excluded from saved fixtures. // The persisted fixture is always the real upstream response, even when @@ -1274,6 +1274,7 @@ export function buildFixtureMatch( endpoint?: EndpointType; turnIndex?: number; hasToolResult?: boolean; + context?: string; } { const match: { userMessage?: string; @@ -1282,6 +1283,7 @@ export function buildFixtureMatch( endpoint?: EndpointType; turnIndex?: number; hasToolResult?: boolean; + context?: string; } = {}; // Include endpoint type for multimedia fixtures @@ -1321,6 +1323,10 @@ export function buildFixtureMatch( match.hasToolResult = messages.some((m) => m.role === "tool"); } + if (request._context) { + match.context = request._context; + } + return match; } diff --git a/src/router.ts b/src/router.ts index ede7598b..5d8861f3 100644 --- a/src/router.ts +++ b/src/router.ts @@ -97,6 +97,13 @@ export function matchFixture( } } + // context — opt-in exact match against the request's _context field. + // If fixture specifies a context, only match requests with that exact context. + // If fixture omits context, match any request regardless of _context. + if (match.context !== undefined) { + if (effective._context !== match.context) continue; + } + // userMessage — case-sensitive match against the last user message content. // String matching is intentionally case-sensitive so fixture authors can // rely on exact string values. This differs from the case-insensitive diff --git a/src/types.ts b/src/types.ts index 997d4355..3c68c7be 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,6 +53,8 @@ export interface ChatCompletionRequest { embeddingInput?: string; /** Endpoint type, set by handlers for fixture endpoint filtering. */ _endpointType?: string; + /** Context identifier, set by handlers for fixture context routing. */ + _context?: string; [key: string]: unknown; } @@ -105,6 +107,7 @@ export interface FixtureMatch { | "realtime" | "realtime-transcription" | "realtime-translation"; + context?: string; } // Fixture response types @@ -406,6 +409,7 @@ export interface FixtureFileEntry { | "realtime" | "realtime-transcription" | "realtime-translation"; + context?: string; // predicate not supported in JSON files }; response: FixtureFileResponse; From a6de21897ebe8f7a2facc9345acf2e12677d8fff Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 18 May 2026 22:36:39 -0700 Subject: [PATCH 2/5] feat: inject X-AIMock-Context in all 21 handler files Every handler that calls matchFixture() now sets _context from the X-AIMock-Context request header. HTTP handlers use getContext(req), WebSocket handlers extract from upgradeHeaders. --- src/bedrock-converse.ts | 3 +++ src/bedrock.ts | 3 +++ src/cohere.ts | 3 +++ src/elevenlabs-audio.ts | 3 +++ src/embeddings.ts | 2 ++ src/fal-audio.ts | 3 +++ src/fal.ts | 2 ++ src/gemini-embeddings.ts | 2 ++ src/gemini-interactions.ts | 2 ++ src/gemini.ts | 2 ++ src/images.ts | 14 ++++++++++---- src/messages.ts | 2 ++ src/ollama.ts | 4 ++++ src/responses.ts | 2 ++ src/server.ts | 2 ++ src/speech.ts | 2 ++ src/transcription.ts | 2 ++ src/video.ts | 2 ++ src/ws-gemini-live.ts | 9 +++++++++ src/ws-realtime.ts | 9 +++++++++ src/ws-responses.ts | 7 +++++++ 21 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index 4077cd97..e9d51315 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -25,6 +25,7 @@ import { isContentWithToolCallsResponse, isErrorResponse, flattenHeaders, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -560,6 +561,7 @@ export async function handleConverse( const completionReq = converseToCompletionRequest(converseReq, modelId, logger); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( @@ -853,6 +855,7 @@ export async function handleConverseStream( const completionReq = converseToCompletionRequest(converseReq, modelId, logger); completionReq.stream = true; completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( diff --git a/src/bedrock.ts b/src/bedrock.ts index 543a91cc..33ae5e56 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -36,6 +36,7 @@ import { isContentWithToolCallsResponse, isErrorResponse, flattenHeaders, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -388,6 +389,7 @@ export async function handleBedrock( // Convert to ChatCompletionRequest for fixture matching const completionReq = bedrockToCompletionRequest(bedrockReq, modelId, logger); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( @@ -1031,6 +1033,7 @@ export async function handleBedrockStream( const completionReq = bedrockToCompletionRequest(bedrockReq, modelId, logger); completionReq.stream = true; completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( diff --git a/src/cohere.ts b/src/cohere.ts index 3461a80b..6caaaf6d 100644 --- a/src/cohere.ts +++ b/src/cohere.ts @@ -37,6 +37,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -833,6 +834,7 @@ export async function handleCohere( // Convert to ChatCompletionRequest for fixture matching const completionReq = cohereToCompletionRequest(cohereReq); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( @@ -1238,6 +1240,7 @@ export async function handleCohereEmbed( messages: [], embeddingInput: combinedInput, _endpointType: "embedding", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/elevenlabs-audio.ts b/src/elevenlabs-audio.ts index d2a60f89..38394315 100644 --- a/src/elevenlabs-audio.ts +++ b/src/elevenlabs-audio.ts @@ -7,6 +7,7 @@ import { serializeErrorResponse, flattenHeaders, FORMAT_TO_CONTENT_TYPE, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -65,6 +66,7 @@ export async function handleElevenLabsTTS( model: (parsed.model_id as string) ?? "eleven_multilingual_v2", messages: [{ role: "user", content: promptText ?? "" }], _endpointType: "elevenlabs-tts", + _context: getContext(req), }; // Validate required field @@ -316,6 +318,7 @@ export async function handleElevenLabsAudio( (subType === "sound-generation" ? "eleven_text_to_sound_v2" : "music_v1"), messages: [{ role: "user", content: promptText ?? "" }], _endpointType: "audio-gen", + _context: getContext(req), }; // Validate required field diff --git a/src/embeddings.ts b/src/embeddings.ts index 9243cbf6..44c0fc74 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -24,6 +24,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -119,6 +120,7 @@ export async function handleEmbeddings( messages: [], embeddingInput: combinedInput, _endpointType: "embedding", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/fal-audio.ts b/src/fal-audio.ts index 46ac27c8..9c4d654f 100644 --- a/src/fal-audio.ts +++ b/src/fal-audio.ts @@ -14,6 +14,7 @@ import { serializeErrorResponse, flattenHeaders, FORMAT_TO_CONTENT_TYPE, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -297,6 +298,7 @@ async function handleQueueSubmit( model: modelId, messages: [{ role: "user", content: prompt }], _endpointType: "fal-audio", + _context: getContext(req), }; const fixture = matchFixture(fixtures, syntheticReq, matchCounts, defaults.requestTransform); @@ -824,6 +826,7 @@ async function handleSyncRun( model: modelId, messages: [{ role: "user", content: prompt }], _endpointType: "fal-audio", + _context: getContext(req), }; const fixture = matchFixture(fixtures, syntheticReq, matchCounts, defaults.requestTransform); diff --git a/src/fal.ts b/src/fal.ts index 315fca06..d9813f20 100644 --- a/src/fal.ts +++ b/src/fal.ts @@ -16,6 +16,7 @@ import { serializeErrorResponse, isJSONResponse, flattenHeaders, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -522,6 +523,7 @@ export async function handleFal( model: modelId, messages: [{ role: "user", content: prompt || JSON.stringify(parsedBody ?? {}) }], _endpointType: "fal", + _context: getContext(req), }; const matchCounts = journal.getFixtureMatchCountsForTest(testId); diff --git a/src/gemini-embeddings.ts b/src/gemini-embeddings.ts index 810015e6..19f70894 100644 --- a/src/gemini-embeddings.ts +++ b/src/gemini-embeddings.ts @@ -26,6 +26,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -117,6 +118,7 @@ export async function handleGeminiEmbedContent( messages: [], embeddingInput: inputText, _endpointType: "embedding", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/gemini-interactions.ts b/src/gemini-interactions.ts index a310179c..86d74461 100644 --- a/src/gemini-interactions.ts +++ b/src/gemini-interactions.ts @@ -28,6 +28,7 @@ import { generateToolCallId, flattenHeaders, getTestId, + getContext, resolveResponse, resolveStrictMode, strictOverrideField, @@ -668,6 +669,7 @@ export async function handleGeminiInteractions( // Convert to ChatCompletionRequest for fixture matching const completionReq = geminiInteractionsToCompletionRequest(interactionsReq); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const streaming = interactionsReq.stream !== false; // default true const model = completionReq.model; diff --git a/src/gemini.ts b/src/gemini.ts index 0b98f362..a10d13ef 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -29,6 +29,7 @@ import { extractOverrides, formatToMime, flattenHeaders, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -632,6 +633,7 @@ export async function handleGemini( // Convert to ChatCompletionRequest for fixture matching const completionReq = geminiToCompletionRequest(geminiReq, model, streaming); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( diff --git a/src/images.ts b/src/images.ts index 48191d0a..b5a7f2a7 100644 --- a/src/images.ts +++ b/src/images.ts @@ -5,6 +5,7 @@ import { isErrorResponse, serializeErrorResponse, flattenHeaders, + getContext, getTestId, resolveResponse, resolveStrictMode, @@ -32,11 +33,16 @@ interface GeminiPredictRequest { [key: string]: unknown; } -function buildSyntheticRequest(model: string, prompt: string): ChatCompletionRequest { +function buildSyntheticRequest( + model: string, + prompt: string, + context?: string, +): ChatCompletionRequest { return { model, messages: [{ role: "user", content: prompt }], _endpointType: "image", + ...(context !== undefined && { _context: context }), }; } @@ -110,7 +116,7 @@ export async function handleImages( return; } - const syntheticReq = buildSyntheticRequest(model, prompt); + const syntheticReq = buildSyntheticRequest(model, prompt, getContext(req)); const testId = getTestId(req); const fixture = matchFixture( fixtures, @@ -342,7 +348,7 @@ export async function handleImageEdit( return; } - const syntheticReq = buildSyntheticRequest(model, prompt); + const syntheticReq = buildSyntheticRequest(model, prompt, getContext(req)); const testId = getTestId(req); const fixture = matchFixture( fixtures, @@ -526,7 +532,7 @@ export async function handleImageVariations( const model = extractFormField(raw, "model", boundary) ?? "dall-e-2"; // Variations don't have a prompt — use a synthetic placeholder for fixture matching - const syntheticReq = buildSyntheticRequest(model, "[variation]"); + const syntheticReq = buildSyntheticRequest(model, "[variation]", getContext(req)); const testId = getTestId(req); const fixture = matchFixture( fixtures, diff --git a/src/messages.ts b/src/messages.ts index 96e70c40..6e0a9ce0 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -31,6 +31,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -747,6 +748,7 @@ export async function handleMessages( // Convert to ChatCompletionRequest for fixture matching const completionReq = claudeToCompletionRequest(claudeReq); + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( diff --git a/src/ollama.ts b/src/ollama.ts index 0491572e..6ee5c7b3 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -35,6 +35,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -539,6 +540,7 @@ export async function handleOllama( // Convert to ChatCompletionRequest for fixture matching const completionReq = ollamaToCompletionRequest(ollamaReq); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( @@ -889,6 +891,7 @@ export async function handleOllamaGenerate( // Convert to ChatCompletionRequest for fixture matching const completionReq = ollamaGenerateToCompletionRequest(generateReq); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( @@ -1212,6 +1215,7 @@ export async function handleOllamaEmbeddings( messages: [], embeddingInput: inputText, _endpointType: "embedding", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/responses.ts b/src/responses.ts index 6560252e..e5b475f5 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -31,6 +31,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -946,6 +947,7 @@ export async function handleResponses( // Convert to ChatCompletionRequest for fixture matching const completionReq = responsesToCompletionRequest(responsesReq); completionReq._endpointType = "chat"; + completionReq._context = getContext(req); const testId = getTestId(req); const fixture = matchFixture( diff --git a/src/server.ts b/src/server.ts index c6e8dda9..8f632085 100644 --- a/src/server.ts +++ b/src/server.ts @@ -36,6 +36,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { handleResponses } from "./responses.js"; import { handleMessages } from "./messages.js"; @@ -486,6 +487,7 @@ async function handleCompletions( // Set endpoint type once early so router/recorder and journal see it body._endpointType = "chat"; + body._context = getContext(req); // Match fixture first — chaos resolution depends on fixture-level overrides // (headers > fixture.chaos > server defaults), so the fixture has to be diff --git a/src/speech.ts b/src/speech.ts index 8ecba2df..22e5b7d0 100644 --- a/src/speech.ts +++ b/src/speech.ts @@ -10,6 +10,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -87,6 +88,7 @@ export async function handleSpeech( model: speechReq.model ?? "tts-1", messages: [{ role: "user", content: speechReq.input }], _endpointType: "speech", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/transcription.ts b/src/transcription.ts index f9a49666..69a8aa11 100644 --- a/src/transcription.ts +++ b/src/transcription.ts @@ -9,6 +9,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -100,6 +101,7 @@ export async function handleTranscription( model, messages: [], _endpointType: endpointType, + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/video.ts b/src/video.ts index ad29427b..dd0f52ff 100644 --- a/src/video.ts +++ b/src/video.ts @@ -9,6 +9,7 @@ import { resolveResponse, resolveStrictMode, strictOverrideField, + getContext, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -166,6 +167,7 @@ export async function handleVideoCreate( model: videoReq.model ?? "sora-2", messages: [{ role: "user", content: videoReq.prompt }], _endpointType: "video", + _context: getContext(req), }; const testId = getTestId(req); diff --git a/src/ws-gemini-live.ts b/src/ws-gemini-live.ts index 94ec26fe..6705659f 100644 --- a/src/ws-gemini-live.ts +++ b/src/ws-gemini-live.ts @@ -363,12 +363,21 @@ async function processMessage( } // Build completion request for fixture matching (include new messages speculatively) + const geminiContextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; + const geminiContext = + typeof geminiContextHeader === "string" + ? geminiContextHeader + : Array.isArray(geminiContextHeader) && geminiContextHeader.length > 0 + ? geminiContextHeader[0] + : undefined; + const completionReq: ChatCompletionRequest = { model: session.model, messages: [...session.conversationHistory, ...newMessages], stream: true, tools: session.tools.length > 0 ? session.tools : undefined, _endpointType: "chat", + _context: geminiContext, }; const testId = defaults.testId ?? DEFAULT_TEST_ID; diff --git a/src/ws-realtime.ts b/src/ws-realtime.ts index 46496c96..489b1645 100644 --- a/src/ws-realtime.ts +++ b/src/ws-realtime.ts @@ -680,10 +680,19 @@ async function handleResponseCreate( }; const endpointType = endpointTypeMap[session.type] ?? "realtime"; + const realtimeContextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; + const realtimeContext = + typeof realtimeContextHeader === "string" + ? realtimeContextHeader + : Array.isArray(realtimeContextHeader) && realtimeContextHeader.length > 0 + ? realtimeContextHeader[0] + : undefined; + const completionReq: ChatCompletionRequest = { model: session.model, messages, _endpointType: endpointType, + _context: realtimeContext, }; const testId = defaults.testId ?? DEFAULT_TEST_ID; diff --git a/src/ws-responses.ts b/src/ws-responses.ts index b8c5f06e..a5f2ea85 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -173,6 +173,13 @@ async function processMessage( const completionReq = responsesToCompletionRequest(responsesReq); completionReq._endpointType = "chat"; + const contextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; + completionReq._context = + typeof contextHeader === "string" + ? contextHeader + : Array.isArray(contextHeader) && contextHeader.length > 0 + ? contextHeader[0] + : undefined; const testId = defaults.testId ?? DEFAULT_TEST_ID; const fixture = matchFixture( fixtures, From 3df00b85e1c5d15ca7b465e00fb2b87358178c43 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 18 May 2026 22:36:45 -0700 Subject: [PATCH 3/5] test: add context routing tests 15 new tests covering matchFixture context matching, getContext header extraction, fixture-loader validation and discriminator, recorder context propagation, and strict-mode interaction. --- src/__tests__/fixture-loader.test.ts | 33 +++++++++++++++++++++++++ src/__tests__/helpers.test.ts | 28 +++++++++++++++++++++ src/__tests__/recorder.test.ts | 25 ++++++++++++++++++- src/__tests__/router.test.ts | 37 ++++++++++++++++++++++++++++ src/__tests__/strict-header.test.ts | 22 +++++++++++++++++ 5 files changed, 144 insertions(+), 1 deletion(-) diff --git a/src/__tests__/fixture-loader.test.ts b/src/__tests__/fixture-loader.test.ts index d62f3c3e..fca5178c 100644 --- a/src/__tests__/fixture-loader.test.ts +++ b/src/__tests__/fixture-loader.test.ts @@ -1006,6 +1006,30 @@ describe("validateFixtures", () => { ).toBe(true); }); + // --- match.context type checks --- + + it("validateFixtures reports error for non-string context", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "test", context: 42 as unknown as string } }), + ]; + const results = validateFixtures(fixtures); + expect( + results.some((r) => r.severity === "error" && r.message.includes("must be a string")), + ).toBe(true); + }); + + it("context is a valid discriminator", () => { + const fixtures = [ + makeFixture({ match: { context: "x" } }), + makeFixture({ match: { userMessage: "hello" } }), + ]; + const results = validateFixtures(fixtures); + const catchAllWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("catch-all"), + ); + expect(catchAllWarnings).toHaveLength(0); + }); + // --- Warning checks --- it("warning: duplicate userMessage", () => { @@ -1545,6 +1569,15 @@ describe("auto-stringify JSON objects in fixture entries", () => { expect((fixture.response as TextResponse).content).toBe("Hello, world!"); }); + it("entryToFixture preserves context field", () => { + const entry: FixtureFileEntry = { + match: { context: "my-ctx", userMessage: "hi" }, + response: { content: "ok" }, + }; + const fixture = entryToFixture(entry); + expect(fixture.match.context).toBe("my-ctx"); + }); + it("passes systemMessage through entryToFixture", () => { const entry: FixtureFileEntry = { match: { userMessage: "test", systemMessage: "name=Atai" }, diff --git a/src/__tests__/helpers.test.ts b/src/__tests__/helpers.test.ts index 9634320c..e11cd540 100644 --- a/src/__tests__/helpers.test.ts +++ b/src/__tests__/helpers.test.ts @@ -1,3 +1,4 @@ +import http from "node:http"; import { describe, it, expect } from "vitest"; import { generateId, @@ -11,6 +12,7 @@ import { buildToolCallChunks, buildTextCompletion, buildToolCallCompletion, + getContext, } from "../helpers.js"; describe("generateId", () => { @@ -292,6 +294,32 @@ describe("buildToolCallChunks", () => { }); }); +describe("getContext", () => { + it("returns header value", () => { + const req = { + headers: { "x-aimock-context": "langgraph-python" }, + } as unknown as http.IncomingMessage; + expect(getContext(req)).toBe("langgraph-python"); + }); + + it("returns undefined when header absent", () => { + const req = { headers: {} } as unknown as http.IncomingMessage; + expect(getContext(req)).toBeUndefined(); + }); + + it("returns first value from array header", () => { + const req = { + headers: { "x-aimock-context": ["first", "second"] }, + } as unknown as http.IncomingMessage; + expect(getContext(req)).toBe("first"); + }); + + it("returns undefined for empty string", () => { + const req = { headers: { "x-aimock-context": "" } } as unknown as http.IncomingMessage; + expect(getContext(req)).toBeUndefined(); + }); +}); + describe("buildTextCompletion", () => { it("returns a valid chat.completion object", () => { const result = buildTextCompletion("Hello!", "gpt-4"); diff --git a/src/__tests__/recorder.test.ts b/src/__tests__/recorder.test.ts index 65498c87..25ae6e64 100644 --- a/src/__tests__/recorder.test.ts +++ b/src/__tests__/recorder.test.ts @@ -5,7 +5,7 @@ import * as os from "node:os"; import * as path from "node:path"; import type { Fixture, FixtureFile } from "../types.js"; import { createServer, type ServerInstance } from "../server.js"; -import { proxyAndRecord, type ProxyCapturedResponse } from "../recorder.js"; +import { proxyAndRecord, buildFixtureMatch, type ProxyCapturedResponse } from "../recorder.js"; import type { RecordConfig } from "../types.js"; import { Logger } from "../logger.js"; import { LLMock } from "../llmock.js"; @@ -4105,6 +4105,29 @@ describe("buildFixtureMatch model recording", () => { }); }); +// --------------------------------------------------------------------------- +// buildFixtureMatch context +// --------------------------------------------------------------------------- + +describe("buildFixtureMatch context", () => { + it("captures _context in match criteria", () => { + const match = buildFixtureMatch({ + model: "gpt-4o", + messages: [{ role: "user", content: "hello" }], + _context: "langgraph-python", + }); + expect(match.context).toBe("langgraph-python"); + }); + + it("omits context when _context is absent", () => { + const match = buildFixtureMatch({ + model: "gpt-4o", + messages: [{ role: "user", content: "hello" }], + }); + expect(match.context).toBeUndefined(); + }); +}); + async function setupUpstreamAndRecorder( upstreamFixtures: Fixture[], providerKey: string = "openai", diff --git a/src/__tests__/router.test.ts b/src/__tests__/router.test.ts index cea9315f..3a2590b8 100644 --- a/src/__tests__/router.test.ts +++ b/src/__tests__/router.test.ts @@ -1107,6 +1107,43 @@ describe("matchFixture — hasToolResult", () => { }); }); +// --------------------------------------------------------------------------- +// matchFixture — context matching +// --------------------------------------------------------------------------- + +describe("matchFixture — context matching", () => { + it("matches fixture with matching context", () => { + const fixture = makeFixture({ context: "foo" }); + const req = makeReq({ _context: "foo" }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("skips fixture with non-matching context", () => { + const fixture = makeFixture({ context: "foo" }); + const req = makeReq({ _context: "bar" }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("matches fixture without context regardless of request context", () => { + const fixture = makeFixture({}); + const req = makeReq({ _context: "bar" }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("skips context fixture when request has no context", () => { + const fixture = makeFixture({ context: "foo" }); + const req = makeReq(); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("context fixture wins over shared when listed first", () => { + const contextual = makeFixture({ context: "foo" }, { content: "contextual" }); + const shared = makeFixture({}, { content: "shared" }); + const req = makeReq({ _context: "foo" }); + expect(matchFixture([contextual, shared], req)).toBe(contextual); + }); +}); + // --------------------------------------------------------------------------- // matchFixture — first-match-wins // --------------------------------------------------------------------------- diff --git a/src/__tests__/strict-header.test.ts b/src/__tests__/strict-header.test.ts index 45e38184..2561f085 100644 --- a/src/__tests__/strict-header.test.ts +++ b/src/__tests__/strict-header.test.ts @@ -220,6 +220,28 @@ describe("X-AIMock-Strict header integration", () => { expect(entries[0].response.strictOverride).toBe(false); }); + it("strict mode 503 fires when context filtering removes all matches", async () => { + const contextFixture: Fixture = { + match: { context: "foo", userMessage: "hello" }, + response: { content: "Hello from foo context" }, + }; + server = await createServer([contextFixture], { port: 0, strict: true }); + + // Context "bar" filters out the only fixture → strict 503 + const mismatched = await httpPost(`${server.url}/v1/chat/completions`, chatRequest("hello"), { + "X-AIMock-Context": "bar", + }); + expect(mismatched.status).toBe(503); + const body503 = JSON.parse(mismatched.body); + expect(body503.error.message).toBe("Strict mode: no fixture matched"); + + // Context "foo" matches → 200 + const matched = await httpPost(`${server.url}/v1/chat/completions`, chatRequest("hello"), { + "X-AIMock-Context": "foo", + }); + expect(matched.status).toBe(200); + }); + it("strict header prevents proxy in record mode", async () => { server = await createServer([], { port: 0, From a75fd3659b296b10720bad160032018825d82d10 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 18 May 2026 22:36:51 -0700 Subject: [PATCH 4/5] docs: add context routing documentation README feature bullet, fixtures page match-field table row and JSON example, multi-turn chooser table row and gotcha, record-replay context-aware recording section, and landing page highlight card. --- README.md | 1 + docs/docs/index.html | 8 ++++++ docs/fixtures/index.html | 50 +++++++++++++++++++++++++++++------ docs/multi-turn/index.html | 23 +++++++++++++++- docs/record-replay/index.html | 48 ++++++++++++++++++++++++++++++++- 5 files changed, 120 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4850bb5a..7bee4d7f 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Run them all on one port with `npx @copilotkit/aimock --config aimock.json`, or - **[MCP](https://aimock.copilotkit.dev/mcp-mock) / [A2A](https://aimock.copilotkit.dev/a2a-mock) / [AG-UI](https://aimock.copilotkit.dev/agui-mock) / [Vector](https://aimock.copilotkit.dev/vector-mock)** — Mock every protocol your AI agents use - **[Chaos Testing](https://aimock.copilotkit.dev/chaos-testing)** — 500 errors, malformed JSON, mid-stream disconnects at any probability - **Per-Request Strict Mode** — `X-AIMock-Strict` header overrides the server-level `--strict` flag per request (`true`/`1` = strict, `false`/`0` = lenient) +- **Context-Based Fixture Routing** — `X-AIMock-Context` header scopes fixtures per integration; fixtures with `match.context` only match requests carrying that context, fixtures without it remain shared - **[Drift Detection](https://aimock.copilotkit.dev/drift-detection)** — Daily CI validation against real APIs - **[Streaming Physics](https://aimock.copilotkit.dev/streaming-physics)** — Configurable `ttft`, `tps`, and `jitter` - **[WebSocket APIs](https://aimock.copilotkit.dev/websocket)** — OpenAI Realtime (GA protocol with models: gpt-realtime, gpt-realtime-2, gpt-realtime-1.5, gpt-realtime-mini; transcription/translation via gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1; image input; commentary phase), Responses WS, Gemini Live diff --git a/docs/docs/index.html b/docs/docs/index.html index 6012c0f2..195010dd 100644 --- a/docs/docs/index.html +++ b/docs/docs/index.html @@ -290,6 +290,14 @@

What's New

One-line CI setup for mock-backed test suites

Setup → + +
+ Context Routing +

+ Scope fixtures per integration with X-AIMock-Context header +

+ Docs → +
diff --git a/docs/fixtures/index.html b/docs/fixtures/index.html index d9189c2e..dd8f089b 100644 --- a/docs/fixtures/index.html +++ b/docs/fixtures/index.html @@ -160,6 +160,16 @@

Match Fields

their own fixture APIs rather than via this field + + context + string + + Restrict to a named context via X-AIMock-Context header. Fixtures with + context only match requests carrying that exact value; fixtures without + context match any request. Same opt-in semantics as + endpoint + + predicate function @@ -242,14 +252,15 @@

5. Validation warnings surface shadowing at load time

duplicate userMessage 'hello' — shadows fixture 0, where 'hello' is the duplicated message and 0 is the zero-based index of the earlier fixture being shadowed. This is advisory, not a hard error: the - check now factors in turnIndex, hasToolResult, and - sequenceIndex when deciding whether two fixtures truly collide, but it does - not consider toolCallId, model, or - predicate, so the warning may still fire when those discriminators are - present. Treat it as advisory: if a runtime differentiator is in place, the fixtures - won't actually shadow each other at match time. Only fixtures with no differentiator at - all will truly shadow on match — that's the case where the second is never reached - because the first wins. Safe to ignore in the former case; investigate in the latter. + check now factors in turnIndex, hasToolResult, + context, and sequenceIndex when deciding whether two fixtures + truly collide, but it does not consider toolCallId, + model, or predicate, so the warning may still fire when those + discriminators are present. Treat it as advisory: if a runtime differentiator is in + place, the fixtures won't actually shadow each other at match time. Only fixtures with + no differentiator at all will truly shadow on match — that's the case where the + second is never reached because the first wins. Safe to ignore in the former case; + investigate in the latter.
  • Catch-all not last — a fixture with an empty match @@ -511,6 +522,29 @@

    From a directory

    +

    Context-scoped fixtures

    +
    +
    + fixtures/context-example.json json +
    + 
    {
+  "fixtures": [
+    {
+      "match": { "userMessage": "hello", "context": "langgraph-python" },
+      "response": { "content": "Hi from LangGraph!" }
+    },
+    {
+      "match": { "userMessage": "hello" },
+      "response": { "content": "Hi from the shared fallback!" }
+    }
+  ]
+}
    +
    +

    + Requests with X-AIMock-Context: langgraph-python match the first fixture; all + other requests fall through to the shared fixture. +

    +

    Programmatically

    programmatic.ts ts
    diff --git a/docs/multi-turn/index.html b/docs/multi-turn/index.html index 922f3409..3b56a788 100644 --- a/docs/multi-turn/index.html +++ b/docs/multi-turn/index.html @@ -291,7 +291,10 @@

    Turn 2 — client runs the tool, sends the result

    -

    Choosing between sequenceIndex, toolCallId, turnIndex, hasToolResult, and predicate

    +

    + Choosing between sequenceIndex, toolCallId, turnIndex, hasToolResult, context, and + predicate +

    Five mechanisms handle different shapes of “the same prompt twice”:

    @@ -341,6 +344,15 @@

    Choosing between sequenceIndex, toolCallId, turnIndex, hasToolResult, and pr pinning a specific tool_call_id. +

    + + + +
    Same user prompt, different response per integration or caller identitycontext + Exact match on the X-AIMock-Context header. Fixtures with + context only match requests carrying that value; fixtures without it + remain shared. Stateless. +
    Arbitrary inspection — message count, specific content at any position, custom @@ -442,6 +454,15 @@

    Gotchas

    instance. See Sequential Responses for when sequenceIndex is the right tool. +
  • + context is an additional discriminator, not a replacement. + context scopes fixtures by integration identity (X-AIMock-Context + header). It combines with all other match fields via AND. Two fixtures with the same + userMessage but different context + values are not duplicates — the validator accounts for this. +
    • diff --git a/docs/record-replay/index.html b/docs/record-replay/index.html index 9b225d15..7c1c76c0 100644 --- a/docs/record-replay/index.html +++ b/docs/record-replay/index.html @@ -556,6 +556,49 @@

    Model-Aware Recording

    }); +

    Context-Aware Recording

    +

    + When a request carries an X-AIMock-Context header, the recorder automatically + captures the context value in match.context. On replay, fixtures with + context only match requests carrying that exact header value — fixtures + without context remain shared across all callers. +

    + +

    Directory routing

    +

    + Without snapshot-style recording (X-Test-Id), recorded fixtures for a given + context are written to a <fixturePath>/<context>/ subdirectory: +

    + +
    +
    + Context directory layout text +
    + 
    fixtures/recorded/
+  openai-2026-05-18T10-30-00-000Z-a1b2c3d4.json     # no context (shared)
+  langgraph-python/
+    openai-2026-05-18T10-30-01-000Z-e5f6a7b8.json   # context = langgraph-python
+  crewai/
+    openai-2026-05-18T10-30-02-000Z-c9d0e1f2.json   # context = crewai
    +
    + +

    + When X-Test-Id is also present, snapshot-style paths take precedence and the + context is captured only in match.context within the fixture file, not in the + directory structure. +

    + +

    Sending X-AIMock-Context

    +
    +
    Header example ts
    + 
    // Set as a default header on your LLM client
+const client = new OpenAI({
+  baseURL: "http://localhost:4010/v1",
+  apiKey: "mock",
+  defaultHeaders: { "X-AIMock-Context": "langgraph-python" },
+});
    +
    +

    Upstream Timeouts

    @@ -858,7 +901,10 @@

    Recording Multi-Turn Conversations// Chat/multimedia — key on the LAST user message only const lastUser = getLastMessageByRole(request.messages, "user"); - return { userMessage: getTextContent(lastUser.content) }; + const match = { userMessage: getTextContent(lastUser.content) }; + // Capture context from X-AIMock-Context header if present + if (request._context) match.context = request._context; + return match; } From 754c48b01b7908cd83ea49929a0738d4f11ba2fa Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 18 May 2026 22:36:57 -0700 Subject: [PATCH 5/5] chore: release v1.26.0 Add context routing and v1.25.0 entries to CHANGELOG. --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceaad092..dd7438ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [1.26.0] - 2026-05-18 + ### Added - **Timing-aware recording and replay** — proxy recording captures per-frame @@ -13,6 +15,12 @@ sources (recorded timings, streaming profiles, global latency). Per-fixture `replaySpeed` override. Covers SSE, NDJSON, Bedrock EventStream, and WebSocket protocols. +- **Context-based fixture routing** — `X-AIMock-Context` header scopes fixtures per integration. Fixtures with `match.context` only match requests carrying that context; fixtures without `context` remain shared. Recorder auto-captures context and routes recorded fixtures into context subdirectories. + +## [1.25.0] - 2026-05-18 + +### Added + - **Gemini `embedContent` endpoint** — `POST /v1beta/models/{model}:embedContent` with deterministic fallback embeddings and fixture matching - **`/v1/images/edit` and `/v1/images/variations` endpoints** — multipart