diff --git a/src/common/config/schemas/index.ts b/src/common/config/schemas/index.ts index ad50609ddc..f1922aad46 100644 --- a/src/common/config/schemas/index.ts +++ b/src/common/config/schemas/index.ts @@ -1,4 +1,5 @@ export * from "./appConfigOnDisk"; +export * from "./modelParameters"; export * from "./providersConfig"; export * from "./configOperations"; export * from "./taskSettings"; diff --git a/src/common/config/schemas/modelParameters.ts b/src/common/config/schemas/modelParameters.ts new file mode 100644 index 0000000000..a1d1a1aa8d --- /dev/null +++ b/src/common/config/schemas/modelParameters.ts @@ -0,0 +1,43 @@ +import type { CallSettings } from "ai"; +import { z } from "zod"; + +const STANDARD_MODEL_PARAMETER_SHAPE = { + max_output_tokens: z.number().int().positive().optional(), + temperature: z.number().min(0).max(2).optional(), + top_p: z.number().min(0).max(1).optional(), + top_k: z.number().int().positive().optional(), + seed: z.number().int().optional(), + frequency_penalty: z.number().optional(), + presence_penalty: z.number().optional(), +} as const; + +export const StandardModelParameterOverridesSchema = z.object(STANDARD_MODEL_PARAMETER_SHAPE); + +// Single runtime mapping source of truth (snake_case config -> AI SDK CallSettings keys) +export const STANDARD_MODEL_PARAMETER_TO_CALL_SETTING = { + max_output_tokens: "maxOutputTokens", + temperature: "temperature", + top_p: "topP", + top_k: "topK", + seed: "seed", + frequency_penalty: "frequencyPenalty", + presence_penalty: "presencePenalty", +} as const satisfies Record; + +export const ModelParameterOverridesSchema = StandardModelParameterOverridesSchema.passthrough(); + +export const ModelParametersByModelSchema = z.record( + z.string().min(1), + ModelParameterOverridesSchema +); + +export type ModelParameterOverrides = z.infer; +export type StandardModelParameterOverrides = z.infer; + +// Downstream type for call settings forwarding — derived from AI SDK CallSettings +export type ResolvedCallSettingsOverrides = Partial< + Pick< + CallSettings, + (typeof STANDARD_MODEL_PARAMETER_TO_CALL_SETTING)[keyof typeof STANDARD_MODEL_PARAMETER_TO_CALL_SETTING] + > +>; diff --git a/src/common/config/schemas/providersConfig.test.ts b/src/common/config/schemas/providersConfig.test.ts index 165c8b638b..155e9f8ae2 100644 --- a/src/common/config/schemas/providersConfig.test.ts +++ b/src/common/config/schemas/providersConfig.test.ts @@ -42,4 +42,82 @@ describe("ProvidersConfigSchema", () => { expect(ProvidersConfigSchema.safeParse(invalid).success).toBe(false); }); + + describe("modelParameters", () => { + it("accepts valid per-model and wildcard overrides", () => { + const valid = { + openai: { + modelParameters: { + "gpt-5": { max_output_tokens: 1024, temperature: 0.4 }, + "*": { top_p: 0.9 }, + }, + }, + }; + + expect(ProvidersConfigSchema.safeParse(valid).success).toBe(true); + }); + + it("rejects negative max_output_tokens", () => { + const invalid = { + openai: { + modelParameters: { + "gpt-5": { max_output_tokens: -1 }, + }, + }, + }; + + expect(ProvidersConfigSchema.safeParse(invalid).success).toBe(false); + }); + + it("rejects temperature values above 2", () => { + const invalid = { + openai: { + modelParameters: { + "gpt-5": { temperature: 3 }, + }, + }, + }; + + expect(ProvidersConfigSchema.safeParse(invalid).success).toBe(false); + }); + + it("rejects top_p values above 1", () => { + const invalid = { + openai: { + modelParameters: { + "gpt-5": { top_p: 1.5 }, + }, + }, + }; + + expect(ProvidersConfigSchema.safeParse(invalid).success).toBe(false); + }); + + it("passes through unknown override keys", () => { + const valid = { + openai: { + modelParameters: { + "gpt-5": { transforms: ["middle-out"] }, + }, + }, + }; + + const parsed = ProvidersConfigSchema.safeParse(valid); + + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.openai?.modelParameters?.["gpt-5"]).toEqual({ + transforms: ["middle-out"], + }); + } + }); + + it("allows provider configs without modelParameters", () => { + const valid = { + openai: { apiKey: "sk-openai-123" }, + }; + + expect(ProvidersConfigSchema.safeParse(valid).success).toBe(true); + }); + }); }); diff --git a/src/common/config/schemas/providersConfig.ts b/src/common/config/schemas/providersConfig.ts index 83b1657bf6..5c9368a203 100644 --- a/src/common/config/schemas/providersConfig.ts +++ b/src/common/config/schemas/providersConfig.ts @@ -1,5 +1,6 @@ import { z } from "zod"; +import { ModelParametersByModelSchema } from "./modelParameters"; import { ProviderModelEntrySchema } from "./providerModelEntry"; export const CacheTtlSchema = z.enum(["5m", "1h"]); @@ -15,6 +16,7 @@ export const BaseProviderConfigSchema = z headers: z.record(z.string(), z.string()).optional(), enabled: z.boolean().optional(), models: z.array(ProviderModelEntrySchema).optional(), + modelParameters: ModelParametersByModelSchema.optional(), }) .passthrough(); diff --git a/src/common/utils/ai/modelParameterOverrides.test.ts b/src/common/utils/ai/modelParameterOverrides.test.ts new file mode 100644 index 0000000000..9a017f5d62 --- /dev/null +++ b/src/common/utils/ai/modelParameterOverrides.test.ts @@ -0,0 +1,433 @@ +import { describe, expect, it } from "bun:test"; +import type { ProvidersConfig } from "@/common/config/schemas/providersConfig"; +import { resolveModelParameterOverrides } from "./modelParameterOverrides"; + +function withAnthropicModelParameters( + modelParameters: Record> +): ProvidersConfig { + return asProvidersConfig({ + anthropic: { + modelParameters, + }, + }); +} + +function withOllamaModelParameters( + modelParameters: Record> +): ProvidersConfig { + return asProvidersConfig({ + ollama: { + modelParameters, + }, + }); +} + +function asProvidersConfig(value: unknown): ProvidersConfig { + return value as ProvidersConfig; +} + +describe("resolveModelParameterOverrides", () => { + it("returns empty standard when providersConfig is null", () => { + const result = resolveModelParameterOverrides(null, "anthropic", "anthropic:claude-sonnet-4-5"); + + expect(result).toEqual({ standard: {} }); + }); + + it("returns empty standard when provider has no modelParameters", () => { + const providersConfig: ProvidersConfig = { + anthropic: {}, + }; + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ standard: {} }); + }); + + it("resolves canonical model match", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + max_output_tokens: 16384, + temperature: 0.7, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + maxOutputTokens: 16384, + temperature: 0.7, + }, + }); + }); + + it("falls back to wildcard entry when model does not have a direct override", () => { + const providersConfig = withAnthropicModelParameters({ + "*": { + max_output_tokens: 4096, + temperature: 0.3, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-opus-4-1" + ); + + expect(result).toEqual({ + standard: { + maxOutputTokens: 4096, + temperature: 0.3, + }, + }); + }); + + it("prefers per-model entry over wildcard", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + max_output_tokens: 8192, + }, + "*": { + max_output_tokens: 2048, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + maxOutputTokens: 8192, + }, + }); + }); + + it("gives effective model entry priority when it differs from canonical model", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + temperature: 0.8, + }, + "claude-sonnet-4-5-20250929": { + temperature: 0.2, + }, + "*": { + temperature: 0.6, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5", + "anthropic:claude-sonnet-4-5-20250929" + ); + + expect(result).toEqual({ + standard: { + temperature: 0.2, + }, + }); + }); + + it("falls back from effective model to canonical model when effective has no override", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + temperature: 0.8, + }, + "*": { + temperature: 0.6, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5", + "anthropic:claude-sonnet-4-5-20250929" + ); + + expect(result).toEqual({ + standard: { + temperature: 0.8, + }, + }); + }); + + it("matches exact colon-suffix model IDs", () => { + const providersConfig = withOllamaModelParameters({ + "gpt-oss:20b": { + temperature: 0.35, + }, + }); + + const result = resolveModelParameterOverrides(providersConfig, "ollama", "ollama:gpt-oss:20b"); + + expect(result).toEqual({ + standard: { + temperature: 0.35, + }, + }); + }); + + it("gives effective model entry priority for colon-suffix model IDs", () => { + const providersConfig = withOllamaModelParameters({ + "gpt-oss:20b": { + temperature: 0.8, + }, + "gpt-oss:120b": { + temperature: 0.2, + }, + "*": { + temperature: 0.6, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "ollama", + "ollama:gpt-oss:20b", + "ollama:gpt-oss:120b" + ); + + expect(result).toEqual({ + standard: { + temperature: 0.2, + }, + }); + }); + + it("falls back to wildcard for colon-suffix model IDs", () => { + const providersConfig = withOllamaModelParameters({ + "*": { + top_p: 0.4, + }, + }); + + const result = resolveModelParameterOverrides(providersConfig, "ollama", "ollama:gpt-oss:20b"); + + expect(result).toEqual({ + standard: { + topP: 0.4, + }, + }); + }); + + it("strips provider prefix from canonical model string", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + top_p: 0.9, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + topP: 0.9, + }, + }); + }); + + it("returns unknown keys as providerExtras while still mapping standard keys", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + transforms: ["middle-out"], + max_output_tokens: 8192, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + maxOutputTokens: 8192, + }, + providerExtras: { + transforms: ["middle-out"], + }, + }); + }); + + it("omits providerExtras when all keys are standard", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + max_output_tokens: 8192, + temperature: 0.4, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + maxOutputTokens: 8192, + temperature: 0.4, + }, + }); + expect(Object.hasOwn(result, "providerExtras")).toBe(false); + }); + + it("ignores out-of-range standard values while keeping valid standard keys", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + max_output_tokens: -1, + top_p: 1.5, + temperature: 0.3, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { + temperature: 0.3, + }, + }); + }); + + it("ignores malformed non-numeric values for standard keys", () => { + const providersConfig = asProvidersConfig({ + anthropic: { + modelParameters: { + "claude-sonnet-4-5": { + max_output_tokens: "not-a-number", + }, + }, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: {}, + }); + }); + + it("ignores malformed string entry for canonical model", () => { + const providersConfig = asProvidersConfig({ + anthropic: { + modelParameters: { + "claude-sonnet-4-5": "bad-entry", + }, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ standard: {} }); + }); + + it("skips malformed effective entry and falls back to canonical plain-object entry", () => { + const providersConfig = asProvidersConfig({ + anthropic: { + modelParameters: { + "claude-sonnet-4-5": { temperature: 0.6 }, + "claude-sonnet-4-5-20250929": "bad-entry", + }, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5", + "anthropic:claude-sonnet-4-5-20250929" + ); + + expect(result).toEqual({ + standard: { temperature: 0.6 }, + }); + }); + + it("ignores NaN and Infinity values for standard keys", () => { + const providersConfig = asProvidersConfig({ + anthropic: { + modelParameters: { + "claude-sonnet-4-5": { + temperature: Number.NaN, + top_p: Number.POSITIVE_INFINITY, + }, + }, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: {}, + }); + }); + + it("returns empty standard when modelParameters exists but is empty", () => { + const providersConfig = withAnthropicModelParameters({}); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ standard: {} }); + }); + + it("treats prototype-collision keys as providerExtras", () => { + const providersConfig = withAnthropicModelParameters({ + "claude-sonnet-4-5": { + toString: "custom-value", + constructor: "another-value", + max_output_tokens: 1024, + }, + }); + + const result = resolveModelParameterOverrides( + providersConfig, + "anthropic", + "anthropic:claude-sonnet-4-5" + ); + + expect(result).toEqual({ + standard: { maxOutputTokens: 1024 }, + providerExtras: { toString: "custom-value", constructor: "another-value" }, + }); + }); +}); diff --git a/src/common/utils/ai/modelParameterOverrides.ts b/src/common/utils/ai/modelParameterOverrides.ts new file mode 100644 index 0000000000..883615f9ab --- /dev/null +++ b/src/common/utils/ai/modelParameterOverrides.ts @@ -0,0 +1,95 @@ +import { + STANDARD_MODEL_PARAMETER_TO_CALL_SETTING, + StandardModelParameterOverridesSchema, + type ResolvedCallSettingsOverrides, +} from "@/common/config/schemas/modelParameters"; +import type { ProvidersConfig } from "@/common/config/schemas/providersConfig"; +import { getModelName } from "@/common/utils/ai/models"; + +export interface ResolvedModelParameterOverrides { + standard: ResolvedCallSettingsOverrides; + providerExtras?: Record; +} + +function isPlainObject(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + const proto: unknown = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +/** + * Resolves model parameter overrides from providers.jsonc config. + * + * Lookup order (first match wins): + * effectiveModelId → canonicalModelId → "*" (wildcard) + * + * Standard keys (max_output_tokens, temperature, etc.) are mapped to AI SDK + * CallSettings names. Unknown keys are returned as providerExtras for merging + * into providerOptions. + */ +export function resolveModelParameterOverrides( + providersConfig: ProvidersConfig | null, + canonicalProviderName: string, + canonicalModelString: string, + effectiveModelString?: string +): ResolvedModelParameterOverrides { + if (!providersConfig) { + return { standard: {} }; + } + + const providerBlock = providersConfig[canonicalProviderName]; + const modelParams = (providerBlock as Record | undefined)?.modelParameters as + | Record + | undefined; + + if (!modelParams) { + return { standard: {} }; + } + + const canonicalModelId = getModelName(canonicalModelString); + const effectiveModelId = + effectiveModelString != null ? getModelName(effectiveModelString) : undefined; + + // Build candidates in precedence order; pick the first that is a valid plain object. + // Malformed entries (strings, arrays, numbers) are silently skipped so the resolver + // falls through to the next candidate rather than iterating junk. + const candidates: unknown[] = [ + effectiveModelId != null && effectiveModelId !== canonicalModelId + ? modelParams[effectiveModelId] + : undefined, + modelParams[canonicalModelId], + modelParams["*"], + ]; + + const entry = candidates.find(isPlainObject); + if (!entry) { + return { standard: {} }; + } + + const standard: Record = {}; + const providerExtras: Record = {}; + + for (const [key, value] of Object.entries(entry)) { + if (Object.hasOwn(STANDARD_MODEL_PARAMETER_TO_CALL_SETTING, key)) { + const standardKey = key as keyof typeof STANDARD_MODEL_PARAMETER_TO_CALL_SETTING; + const sdkKey = STANDARD_MODEL_PARAMETER_TO_CALL_SETTING[standardKey]; + + // Config may be hand-edited; validate each standard key against schema bounds defensively. + const validator = StandardModelParameterOverridesSchema.shape[standardKey]; + const parsed = validator.safeParse(value); + if (parsed.success && parsed.data !== undefined) { + standard[sdkKey] = parsed.data; + } + continue; + } + + providerExtras[key] = value; + } + + return { + standard: standard as ResolvedCallSettingsOverrides, + ...(Object.keys(providerExtras).length > 0 ? { providerExtras } : {}), + }; +} diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index 30f0f40207..8fe30882cc 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -41,6 +41,7 @@ import * as agentResolution from "./agentResolution"; import * as streamContextBuilder from "./streamContextBuilder"; import * as messagePipeline from "./messagePipeline"; import * as toolsModule from "@/common/utils/tools/tools"; +import * as providerOptionsModule from "@/common/utils/ai/providerOptions"; import * as systemMessageModule from "./systemMessage"; describe("AIService", () => { @@ -1194,6 +1195,454 @@ describe("AIService.streamMessage compaction boundary slicing", () => { }); }); +describe("AIService.streamMessage model parameter overrides", () => { + const ANTHROPIC_MODEL = "anthropic:claude-sonnet-4-5"; + + interface ModelParameterOverridesHarness { + service: AIService; + config: Config; + startStreamCalls: unknown[][]; + } + + function createWorkspaceMetadata(workspaceId: string, projectPath: string): WorkspaceMetadata { + return { + id: workspaceId, + name: "workspace-model-overrides", + projectName: "project-model-overrides", + projectPath, + runtimeConfig: { type: "local" }, + }; + } + + function providerOptionsFromStartStreamCall(startStreamArgs: unknown[]): Record { + const providerOptions = startStreamArgs[11]; + if (!providerOptions || typeof providerOptions !== "object" || Array.isArray(providerOptions)) { + throw new Error("Expected provider options object at startStream arg index 11"); + } + + return providerOptions as Record; + } + + function callSettingsOverridesFromStartStreamCall( + startStreamArgs: unknown[] + ): Record { + const callSettingsOverrides = startStreamArgs[21]; + if ( + !callSettingsOverrides || + typeof callSettingsOverrides !== "object" || + Array.isArray(callSettingsOverrides) + ) { + throw new Error("Expected call settings overrides object at startStream arg index 21"); + } + + return callSettingsOverrides as Record; + } + + function createHarness( + muxHomePath: string, + metadata: WorkspaceMetadata + ): ModelParameterOverridesHarness { + const config = new Config(muxHomePath); + const historyService = new HistoryService(config); + const initStateManager = new InitStateManager(config); + const providerService = new ProviderService(config); + const service = new AIService(config, historyService, initStateManager, providerService); + + const startStreamCalls: unknown[][] = []; + + const resolvedAgentResult: Awaited> = { + success: true, + data: { + effectiveAgentId: "exec", + agentDefinition: { + id: "exec", + scope: "built-in", + frontmatter: { name: "Exec" }, + body: "Exec agent body", + }, + agentDiscoveryPath: metadata.projectPath, + isSubagentWorkspace: false, + agentIsPlanLike: false, + effectiveMode: "exec", + taskSettings: DEFAULT_TASK_SETTINGS, + taskDepth: 0, + shouldDisableTaskToolsForDepth: false, + effectiveToolPolicy: undefined, + toolNamesForSentinel: [], + }, + }; + spyOn(agentResolution, "resolveAgentForStream").mockResolvedValue(resolvedAgentResult); + + spyOn(streamContextBuilder, "buildPlanInstructions").mockResolvedValue({ + effectiveAdditionalInstructions: undefined, + planFilePath: path.join(metadata.projectPath, "plan.md"), + planContentForTransition: undefined, + }); + + spyOn(streamContextBuilder, "buildStreamSystemContext").mockResolvedValue({ + agentSystemPrompt: "test-agent-prompt", + systemMessage: "test-system-message", + systemMessageTokens: 1, + agentDefinitions: undefined, + availableSkills: undefined, + }); + + spyOn(messagePipeline, "prepareMessagesForProvider").mockImplementation((args) => { + const preparedMessages = args.messagesWithSentinel as unknown as Awaited< + ReturnType + >; + return Promise.resolve(preparedMessages); + }); + + spyOn(toolsModule, "getToolsForModel").mockResolvedValue({}); + spyOn(systemMessageModule, "readToolInstructions").mockResolvedValue({}); + + const fakeModel = Object.create(null) as LanguageModel; + const providerModelFactory = Reflect.get(service, "providerModelFactory") as + | ProviderModelFactory + | undefined; + if (!providerModelFactory) { + throw new Error("Expected AIService.providerModelFactory in streamMessage test harness"); + } + + const resolveAndCreateModelResult: Awaited< + ReturnType + > = { + success: true, + data: { + model: fakeModel, + effectiveModelString: ANTHROPIC_MODEL, + canonicalModelString: ANTHROPIC_MODEL, + canonicalProviderName: "anthropic", + canonicalModelId: "claude-sonnet-4-5", + routedThroughGateway: false, + }, + }; + spyOn(providerModelFactory, "resolveAndCreateModel").mockResolvedValue( + resolveAndCreateModelResult + ); + + spyOn(service, "getWorkspaceMetadata").mockResolvedValue({ + success: true, + data: metadata, + }); + + spyOn(initStateManager, "waitForInit").mockResolvedValue(undefined); + + spyOn(config, "findWorkspace").mockReturnValue({ + workspacePath: metadata.projectPath, + projectPath: metadata.projectPath, + }); + + spyOn(historyService, "commitPartial").mockResolvedValue({ + success: true, + data: undefined, + }); + + spyOn(historyService, "appendToHistory").mockImplementation((_workspaceId, message) => { + message.metadata = { + ...(message.metadata ?? {}), + historySequence: 9, + }; + + return Promise.resolve({ success: true, data: undefined }); + }); + + const streamManager = (service as unknown as { streamManager: StreamManager }).streamManager; + const streamToken = "stream-token" as ReturnType; + + spyOn(streamManager, "generateStreamToken").mockReturnValue(streamToken); + spyOn(streamManager, "createTempDirForStream").mockResolvedValue( + path.join(metadata.projectPath, ".tmp-stream") + ); + spyOn(streamManager, "isResponseIdLost").mockReturnValue(false); + spyOn(streamManager, "startStream").mockImplementation((...args: unknown[]) => { + startStreamCalls.push(args); + + const startStreamResult: Awaited> = { + success: true, + data: streamToken, + }; + + return Promise.resolve(startStreamResult); + }); + + return { + service, + config, + startStreamCalls, + }; + } + + async function streamAndGetStartStreamArgs( + harness: ModelParameterOverridesHarness, + workspaceId: string, + modelString = ANTHROPIC_MODEL + ): Promise { + const result = await harness.service.streamMessage({ + messages: [createMuxMessage("user-message", "user", "hello")], + workspaceId, + modelString, + thinkingLevel: "off", + }); + + expect(result.success).toBe(true); + expect(harness.startStreamCalls).toHaveLength(1); + + const startStreamCall = harness.startStreamCalls[0]; + if (!startStreamCall) { + throw new Error("Expected streamManager.startStream call arguments"); + } + + return startStreamCall; + } + + afterEach(() => { + mock.restore(); + }); + + it("passes resolved call settings overrides as the final startStream argument", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-standard"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-standard"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({ + anthropic: { + modelParameters: { + "claude-sonnet-4-5": { + max_output_tokens: 16384, + temperature: 0.7, + }, + }, + }, + }); + + const startStreamArgs = await streamAndGetStartStreamArgs(harness, workspaceId); + expect(callSettingsOverridesFromStartStreamCall(startStreamArgs)).toEqual({ + maxOutputTokens: 16384, + temperature: 0.7, + }); + }); + + it("deep-merges provider extras under Mux-built provider options", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-provider-extras"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-provider-extras"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({ + anthropic: { + modelParameters: { + "*": { + custom_knob: 40, + }, + }, + }, + }); + + spyOn(providerOptionsModule, "buildProviderOptions").mockReturnValue({ + anthropic: { + thinking: { type: "enabled" }, + }, + }); + + const startStreamArgs = await streamAndGetStartStreamArgs(harness, workspaceId); + expect(providerOptionsFromStartStreamCall(startStreamArgs)).toEqual({ + anthropic: { + custom_knob: 40, + thinking: { type: "enabled" }, + }, + }); + }); + + it("passes empty call settings overrides when providers config is empty", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-empty"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-empty"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({}); + + const startStreamArgs = await streamAndGetStartStreamArgs(harness, workspaceId); + expect(startStreamArgs[21]).toEqual({}); + }); + + it("preserves Mux-built provider options when provider extras conflict", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-conflict"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-conflict"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({ + anthropic: { + modelParameters: { + "*": { + thinking: { type: "disabled" }, + custom_knob: 10, + }, + }, + }, + }); + + spyOn(providerOptionsModule, "buildProviderOptions").mockReturnValue({ + anthropic: { + thinking: { type: "enabled" }, + sendReasoning: true, + }, + }); + + const startStreamArgs = await streamAndGetStartStreamArgs(harness, workspaceId); + expect(providerOptionsFromStartStreamCall(startStreamArgs)).toEqual({ + anthropic: { + custom_knob: 10, + thinking: { type: "enabled" }, + sendReasoning: true, + }, + }); + }); + + it("deep-merges nested provider extras with Mux-built options", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-nested"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-nested"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + // Override to OpenRouter provider + const providerModelFactory = Reflect.get( + harness.service, + "providerModelFactory" + ) as ProviderModelFactory; + const fakeModel = Object.create(null) as LanguageModel; + spyOn(providerModelFactory, "resolveAndCreateModel").mockResolvedValue({ + success: true, + data: { + model: fakeModel, + effectiveModelString: "openrouter:deepseek/deepseek-r1", + canonicalModelString: "openrouter:deepseek/deepseek-r1", + canonicalProviderName: "openrouter", + canonicalModelId: "deepseek/deepseek-r1", + routedThroughGateway: false, + }, + }); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({ + openrouter: { + modelParameters: { + "*": { + reasoning: { max_tokens: 4096 }, + }, + }, + }, + }); + + spyOn(providerOptionsModule, "buildProviderOptions").mockReturnValue({ + openrouter: { + reasoning: { + enabled: true, + effort: "high", + exclude: false, + }, + }, + }); + + const startStreamArgs = await streamAndGetStartStreamArgs( + harness, + workspaceId, + "openrouter:deepseek/deepseek-r1" + ); + expect(providerOptionsFromStartStreamCall(startStreamArgs)).toEqual({ + openrouter: { + reasoning: { + max_tokens: 4096, + enabled: true, + effort: "high", + exclude: false, + }, + }, + }); + }); + + it("Mux values win on nested leaf conflicts during deep merge", async () => { + using muxHome = new DisposableTempDir("ai-service-model-overrides-nested-conflict"); + const projectPath = path.join(muxHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-model-overrides-nested-conflict"; + const metadata = createWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(muxHome.path, metadata); + + // Override to OpenRouter provider + const providerModelFactory = Reflect.get( + harness.service, + "providerModelFactory" + ) as ProviderModelFactory; + const fakeModel = Object.create(null) as LanguageModel; + spyOn(providerModelFactory, "resolveAndCreateModel").mockResolvedValue({ + success: true, + data: { + model: fakeModel, + effectiveModelString: "openrouter:deepseek/deepseek-r1", + canonicalModelString: "openrouter:deepseek/deepseek-r1", + canonicalProviderName: "openrouter", + canonicalModelId: "deepseek/deepseek-r1", + routedThroughGateway: false, + }, + }); + + spyOn(harness.config, "loadProvidersConfig").mockReturnValue({ + openrouter: { + modelParameters: { + "*": { + reasoning: { enabled: false, max_tokens: 4096 }, + }, + }, + }, + }); + + spyOn(providerOptionsModule, "buildProviderOptions").mockReturnValue({ + openrouter: { + reasoning: { + enabled: true, + effort: "high", + exclude: false, + }, + }, + }); + + const startStreamArgs = await streamAndGetStartStreamArgs( + harness, + workspaceId, + "openrouter:deepseek/deepseek-r1" + ); + expect(providerOptionsFromStartStreamCall(startStreamArgs)).toEqual({ + openrouter: { + reasoning: { + max_tokens: 4096, + enabled: true, + effort: "high", + exclude: false, + }, + }, + }); + }); +}); + describe("normalizeAnthropicBaseURL", () => { it("appends /v1 to URLs without it", () => { expect(normalizeAnthropicBaseURL("https://api.anthropic.com")).toBe( diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 22ac3556f0..6a8729685e 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -52,6 +52,7 @@ import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpSer import { WorkspaceMcpOverridesService } from "./workspaceMcpOverridesService"; import type { TaskService } from "@/node/services/taskService"; import { buildProviderOptions, buildRequestHeaders } from "@/common/utils/ai/providerOptions"; +import { resolveModelParameterOverrides } from "@/common/utils/ai/modelParameterOverrides"; import { sliceMessagesFromLatestCompactionBoundary } from "@/common/utils/messages/compactionBoundary"; import { THINKING_LEVEL_OFF, type ThinkingLevel } from "@/common/types/thinking"; @@ -117,6 +118,36 @@ function safeClone(value: T): T { : (JSON.parse(JSON.stringify(value)) as T); } +/** Plain-object guard for recursive provider option merging. */ +function isProviderPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value == null || Array.isArray(value)) { + return false; + } + const proto: unknown = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +/** + * Recursively merge user-provided provider extras under Mux-built provider options. + * Mux values win on leaf conflicts; both sides' non-conflicting nested fields are preserved. + */ +function mergeProviderExtrasUnderMux( + providerExtras: Record, + muxProviderNamespace: Record +): Record { + const merged: Record = { ...providerExtras }; + + for (const [key, muxValue] of Object.entries(muxProviderNamespace)) { + const extraValue = merged[key]; + merged[key] = + isProviderPlainObject(extraValue) && isProviderPlainObject(muxValue) + ? mergeProviderExtrasUnderMux(extraValue, muxValue) + : muxValue; + } + + return merged; +} + interface ToolExecutionContext { toolCallId?: string; abortSignal?: AbortSignal; @@ -1039,6 +1070,38 @@ export class AIService extends EventEmitter { this.providerService.getConfig() ); + // --- Model parameter overrides from providers.jsonc --- + const providersConfig = this.config.loadProvidersConfig(); + const resolvedOverrides = resolveModelParameterOverrides( + providersConfig, + canonicalProviderName, + canonicalModelString, + effectiveModelString + ); + + // Merge provider extras (user knobs) UNDER Mux-built options (safety-critical). + // Recursive merge within the provider namespace preserves non-conflicting nested + // subfields (e.g., user reasoning.max_tokens alongside Mux reasoning.enabled). + // Mux-built values win on leaf conflicts for safety of thinking/reasoning/cache. + const muxProviderNamespace = (providerOptions as Record)?.[ + canonicalProviderName + ]; + const mergedProviderOptions = resolvedOverrides.providerExtras + ? { + ...providerOptions, + [canonicalProviderName]: isProviderPlainObject(muxProviderNamespace) + ? mergeProviderExtrasUnderMux(resolvedOverrides.providerExtras, muxProviderNamespace) + : resolvedOverrides.providerExtras, + } + : providerOptions; + + if (Object.keys(resolvedOverrides.standard).length > 0 || resolvedOverrides.providerExtras) { + log.debug( + `Resolved model parameter overrides for ${canonicalModelString}`, + resolvedOverrides + ); + } + // Debug dump: Log the complete LLM request when MUX_DEBUG_LLM_REQUEST is set if (process.env.MUX_DEBUG_LLM_REQUEST === "1") { log.info( @@ -1054,7 +1117,7 @@ export class AIService extends EventEmitter { { description: t.description, inputSchema: t.inputSchema }, ]) ), - providerOptions, + providerOptions: mergedProviderOptions, thinkingLevel: effectiveThinkingLevel, maxOutputTokens, mode: effectiveMode, @@ -1065,6 +1128,16 @@ export class AIService extends EventEmitter { 2 )}` ); + + if (resolvedOverrides.standard && Object.keys(resolvedOverrides.standard).length > 0) { + log.debug("Model parameter overrides (standard):", resolvedOverrides.standard); + } + if (resolvedOverrides.providerExtras) { + log.debug( + "Model parameter overrides (provider extras):", + resolvedOverrides.providerExtras + ); + } } if (combinedAbortSignal.aborted) { @@ -1141,7 +1214,7 @@ export class AIService extends EventEmitter { ...(acpPromptId != null ? { acpPromptId } : {}), ...(modelCostsIncluded(modelResult.data.model) ? { costsIncluded: true } : {}), }, - providerOptions, + mergedProviderOptions, maxOutputTokens, effectiveToolPolicy, streamToken, // Pass the pre-generated stream token @@ -1150,7 +1223,8 @@ export class AIService extends EventEmitter { effectiveThinkingLevel, requestHeaders, effectiveMuxProviderOptions.anthropic?.cacheTtl ?? undefined, - forceToolChoice + forceToolChoice, + resolvedOverrides.standard ); if (!streamResult.success) { diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 72b7e2e07f..6118633798 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1,10 +1,12 @@ -import { describe, test, expect, afterEach, beforeEach } from "bun:test"; +import { describe, test, expect, afterEach, beforeEach, mock, spyOn } from "bun:test"; import * as fs from "node:fs/promises"; import { KNOWN_MODELS } from "@/common/constants/knownModels"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { StreamManager, stripEncryptedContent } from "./streamManager"; +import * as aiSdk from "ai"; import { APICallError, RetryError, type ModelMessage } from "ai"; +import * as modelStatsModule from "@/common/utils/tokens/modelStats"; import type { HistoryService } from "./historyService"; import { createTestHistoryService } from "./testHistoryService"; import { createAnthropic } from "@ai-sdk/anthropic"; @@ -374,6 +376,7 @@ describe("StreamManager - stopWhen configuration", () => { { switch_agent: {} }, undefined, undefined, + undefined, [{ regex_match: "switch_agent", action: "require" }], true, () => false, @@ -438,6 +441,176 @@ describe("StreamManager - stopWhen configuration", () => { }); }); +describe("StreamManager - call settings overrides", () => { + interface StreamRequestConfigForTests { + model: unknown; + messages: ModelMessage[]; + system?: string; + tools?: Record; + providerOptions?: Record; + headers?: Record; + maxOutputTokens?: number; + streamCallSettings?: Record; + } + + type BuildStreamRequestConfig = (...args: unknown[]) => StreamRequestConfigForTests; + type CreateStreamResult = ( + request: StreamRequestConfigForTests, + abortController: AbortController + ) => unknown; + + const model = createAnthropic({ apiKey: "test" })("claude-sonnet-4-5"); + const modelString = KNOWN_MODELS.SONNET.id; + const messages: ModelMessage[] = [{ role: "user", content: "hello" }]; + + function getRequestHelpers(streamManager: StreamManager): { + buildRequestConfig: BuildStreamRequestConfig; + createStreamResult: CreateStreamResult; + } { + const buildRequestConfig = Reflect.get(streamManager, "buildStreamRequestConfig") as + | BuildStreamRequestConfig + | undefined; + const createStreamResultMethod = Reflect.get(streamManager, "createStreamResult") as + | CreateStreamResult + | undefined; + + expect(typeof buildRequestConfig).toBe("function"); + expect(typeof createStreamResultMethod).toBe("function"); + + if (!buildRequestConfig || !createStreamResultMethod) { + throw new Error("Expected StreamManager private helpers to exist"); + } + + return { + buildRequestConfig, + createStreamResult: (request, abortController) => + createStreamResultMethod.call(streamManager, request, abortController), + }; + } + + function buildRequest( + buildRequestConfig: BuildStreamRequestConfig, + options: { + maxOutputTokens?: number; + callSettingsOverrides?: { + maxOutputTokens?: number; + temperature?: number; + topP?: number; + }; + } + ): StreamRequestConfigForTests { + return buildRequestConfig( + model, + modelString, + messages, + "system", + undefined, + undefined, + options.maxOutputTokens, + options.callSettingsOverrides, + undefined, + false, + undefined, + undefined, + undefined + ); + } + + function setupStreamTextSpy() { + return spyOn(aiSdk, "streamText").mockReturnValue({ + fullStream: (async function* asyncGenerator() { + yield* [] as unknown[]; + await Promise.resolve(); + })(), + usage: Promise.resolve(undefined), + providerMetadata: Promise.resolve(undefined), + totalUsage: Promise.resolve(undefined), + steps: Promise.resolve([]), + } as unknown as ReturnType); + } + + afterEach(() => { + mock.restore(); + }); + + test("uses config maxOutputTokens override when explicit maxOutputTokens is missing", () => { + const streamManager = new StreamManager(historyService); + const { buildRequestConfig, createStreamResult } = getRequestHelpers(streamManager); + const streamTextSpy = setupStreamTextSpy(); + + spyOn(modelStatsModule, "getModelStats").mockReturnValue({ + max_input_tokens: 200000, + max_output_tokens: 8192, + input_cost_per_token: 0, + output_cost_per_token: 0, + }); + + const request = buildRequest(buildRequestConfig, { + callSettingsOverrides: { maxOutputTokens: 4096 }, + }); + + createStreamResult(request, new AbortController()); + + expect(streamTextSpy).toHaveBeenCalledWith(expect.objectContaining({ maxOutputTokens: 4096 })); + }); + + test("uses explicit maxOutputTokens over config maxOutputTokens override", () => { + const streamManager = new StreamManager(historyService); + const { buildRequestConfig, createStreamResult } = getRequestHelpers(streamManager); + const streamTextSpy = setupStreamTextSpy(); + + spyOn(modelStatsModule, "getModelStats").mockReturnValue({ + max_input_tokens: 200000, + max_output_tokens: 8192, + input_cost_per_token: 0, + output_cost_per_token: 0, + }); + + const request = buildRequest(buildRequestConfig, { + maxOutputTokens: 1024, + callSettingsOverrides: { maxOutputTokens: 4096 }, + }); + + createStreamResult(request, new AbortController()); + + expect(streamTextSpy).toHaveBeenCalledWith(expect.objectContaining({ maxOutputTokens: 1024 })); + }); + + test("forwards stream call settings to streamText", () => { + const streamManager = new StreamManager(historyService); + const { buildRequestConfig, createStreamResult } = getRequestHelpers(streamManager); + const streamTextSpy = setupStreamTextSpy(); + + const request = buildRequest(buildRequestConfig, { + callSettingsOverrides: { temperature: 0.5, topP: 0.9 }, + }); + + createStreamResult(request, new AbortController()); + + expect(streamTextSpy).toHaveBeenCalledWith( + expect.objectContaining({ + temperature: 0.5, + topP: 0.9, + }) + ); + }); + + test("does not store streamCallSettings when overrides are empty", () => { + const streamManager = new StreamManager(historyService); + const { buildRequestConfig } = getRequestHelpers(streamManager); + + const requestWithUndefined = buildRequest(buildRequestConfig, { + callSettingsOverrides: undefined, + }); + const requestWithEmpty = buildRequest(buildRequestConfig, { + callSettingsOverrides: {}, + }); + + expect(requestWithUndefined.streamCallSettings).toBeUndefined(); + expect(requestWithEmpty.streamCallSettings).toBeUndefined(); + }); +}); + describe("StreamManager - stripEncryptedContent", () => { test("strips encryptedContent from array output shape", () => { const output = [ diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 6085ce75f2..edabdfb18a 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -56,6 +56,7 @@ import { extractToolMediaAsUserMessagesFromModelMessages } from "@/node/utils/me import { normalizeGatewayModel } from "@/common/utils/ai/models"; import { MUX_GATEWAY_SESSION_EXPIRED_MESSAGE } from "@/common/constants/muxGatewayOAuth"; import { getModelStats, getModelStatsResolved } from "@/common/utils/tokens/modelStats"; +import type { ResolvedCallSettingsOverrides } from "@/common/config/schemas/modelParameters"; import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries"; import { getErrorMessage } from "@/common/utils/errors"; import { shellQuote } from "@/common/utils/shell"; @@ -105,6 +106,7 @@ interface StreamRequestConfig { /** Per-request HTTP headers (e.g., anthropic-beta for 1M context). */ headers?: Record; maxOutputTokens?: number; + streamCallSettings?: Omit; hasQueuedMessage?: () => boolean; toolPolicy?: ToolPolicy; // Belt-and-suspenders for top-level agents: force the model to call the @@ -1057,6 +1059,7 @@ export class StreamManager extends EventEmitter { tools?: Record, providerOptions?: Record, maxOutputTokens?: number, + callSettingsOverrides?: ResolvedCallSettingsOverrides, toolPolicy?: ToolPolicy, forceToolChoice?: boolean, hasQueuedMessage?: () => boolean, @@ -1092,11 +1095,15 @@ export class StreamManager extends EventEmitter { // a custom model's provider may not support the mapped model's output cap. // If no metadata exists, omit the parameter to let the provider use its // default (Anthropic requires this but has low defaults). + const { maxOutputTokens: configMaxOutputTokens, ...streamCallSettings } = + callSettingsOverrides ?? {}; + const runtimeModelStats = getModelStats(modelString); // Fall back to resolved stats for custom aliases (e.g., provider alias mappedToModel). const resolvedModelStats = runtimeModelStats ?? getModelStatsResolved(modelString, this.getProvidersConfig()); - const effectiveMaxOutputTokens = maxOutputTokens ?? resolvedModelStats?.max_output_tokens; + const effectiveMaxOutputTokens = + maxOutputTokens ?? configMaxOutputTokens ?? resolvedModelStats?.max_output_tokens; let toolChoice: StreamRequestConfig["toolChoice"] | undefined; if (forceToolChoice && toolPolicy && finalTools) { @@ -1150,6 +1157,8 @@ export class StreamManager extends EventEmitter { providerOptions: finalProviderOptions, headers, maxOutputTokens: effectiveMaxOutputTokens, + streamCallSettings: + Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, hasQueuedMessage, toolPolicy, toolChoice, @@ -1242,6 +1251,7 @@ export class StreamManager extends EventEmitter { providerOptions: request.providerOptions as any, // Pass provider-specific options (thinking/reasoning config) headers: request.headers, // Per-request HTTP headers (e.g., anthropic-beta for 1M context) maxOutputTokens: request.maxOutputTokens, + ...(request.streamCallSettings ?? {}), }); } @@ -1266,6 +1276,7 @@ export class StreamManager extends EventEmitter { maxOutputTokens?: number, toolPolicy?: ToolPolicy, forceToolChoice?: boolean, + callSettingsOverrides?: ResolvedCallSettingsOverrides, hasQueuedMessage?: () => boolean, workspaceName?: string, thinkingLevel?: string, @@ -1284,6 +1295,7 @@ export class StreamManager extends EventEmitter { tools, providerOptions, maxOutputTokens, + callSettingsOverrides, toolPolicy, forceToolChoice, hasQueuedMessage, @@ -2678,7 +2690,8 @@ export class StreamManager extends EventEmitter { thinkingLevel?: string, headers?: Record, anthropicCacheTtlOverride?: AnthropicCacheTtl, - forceToolChoice?: boolean + forceToolChoice?: boolean, + callSettingsOverrides?: ResolvedCallSettingsOverrides ): Promise> { const typedWorkspaceId = workspaceId as WorkspaceId; @@ -2752,6 +2765,7 @@ export class StreamManager extends EventEmitter { maxOutputTokens, toolPolicy, forceToolChoice, + callSettingsOverrides, hasQueuedMessage, workspaceName, thinkingLevel,