diff --git a/packages/app/src/components/dialog-select-provider.tsx b/packages/app/src/components/dialog-select-provider.tsx index 89310286daf2..2b0fa888e4e0 100644 --- a/packages/app/src/components/dialog-select-provider.tsx +++ b/packages/app/src/components/dialog-select-provider.tsx @@ -24,6 +24,7 @@ export const DialogSelectProvider: Component = () => { if (id === "openai") return language.t("dialog.provider.openai.note") if (id.startsWith("github-copilot")) return language.t("dialog.provider.copilot.note") if (id === "opencode-go") return language.t("dialog.provider.opencodeGo.tagline") + if (id === "litellm") return language.t("dialog.provider.litellm.note") } return ( diff --git a/packages/app/src/components/settings-providers.tsx b/packages/app/src/components/settings-providers.tsx index ef8d03aedd0a..e8e62e31679d 100644 --- a/packages/app/src/components/settings-providers.tsx +++ b/packages/app/src/components/settings-providers.tsx @@ -26,6 +26,7 @@ const PROVIDER_NOTES = [ { match: (id: string) => id === "google", key: "dialog.provider.google.note" }, { match: (id: string) => id === "openrouter", key: "dialog.provider.openrouter.note" }, { match: (id: string) => id === "vercel", key: "dialog.provider.vercel.note" }, + { match: (id: string) => id === "litellm", key: "dialog.provider.litellm.note" }, ] as const export const SettingsProviders: Component = () => { diff --git a/packages/app/src/i18n/en.ts b/packages/app/src/i18n/en.ts index 76c96dcc02af..d509866f485d 100644 --- a/packages/app/src/i18n/en.ts +++ b/packages/app/src/i18n/en.ts @@ -110,6 +110,7 @@ export const dict = { "dialog.provider.google.note": "Gemini models for fast, structured responses", "dialog.provider.openrouter.note": "Access all supported models from one provider", "dialog.provider.vercel.note": "Unified access to AI models with smart routing", + "dialog.provider.litellm.note": "Unified proxy for 100+ LLMs with load balancing and spend tracking", "dialog.model.select.title": "Select model", "dialog.model.search.placeholder": "Search models", diff --git a/packages/core/src/provider.ts b/packages/core/src/provider.ts index 044cf7b16b08..f12cfddd18a2 100644 --- a/packages/core/src/provider.ts +++ b/packages/core/src/provider.ts @@ -18,6 +18,7 @@ export const ID = Schema.String.pipe( openrouter: schema.make("openrouter"), mistral: schema.make("mistral"), gitlab: schema.make("gitlab"), + litellm: schema.make("litellm"), })), ) export type ID = typeof ID.Type diff --git a/packages/opencode/src/plugin/index.ts b/packages/opencode/src/plugin/index.ts index 478114209207..ec1b73259235 100644 --- a/packages/opencode/src/plugin/index.ts +++ b/packages/opencode/src/plugin/index.ts @@ -13,6 +13,7 @@ import { CodexAuthPlugin } from "./openai/codex" import { Session } from "@/session/session" import { NamedError } from "@opencode-ai/core/util/error" import { CopilotAuthPlugin } from "./github-copilot/copilot" +import { LiteLLMPlugin } from "./litellm/litellm" import { gitlabAuthPlugin as GitlabAuthPlugin } from "opencode-gitlab-auth" import { PoeAuthPlugin } from "opencode-poe-auth" import { CloudflareAIGatewayAuthPlugin, CloudflareWorkersAuthPlugin } from "./cloudflare" @@ -71,6 +72,7 @@ function internalPlugins(flags: RuntimeFlags.Info): PluginInstance[] { experimentalWebSockets: experimentalWebSocketsEnabled({ enabled: flags.experimentalWebSockets }), }), CopilotAuthPlugin, + LiteLLMPlugin, GitlabAuthPlugin, PoeAuthPlugin, CloudflareWorkersAuthPlugin, diff --git a/packages/opencode/src/plugin/litellm/litellm.ts b/packages/opencode/src/plugin/litellm/litellm.ts new file mode 100644 index 000000000000..56af13023ea9 --- /dev/null +++ b/packages/opencode/src/plugin/litellm/litellm.ts @@ -0,0 +1,66 @@ +import type { Hooks, PluginInput } from "@opencode-ai/plugin" +import * as Log from "@opencode-ai/core/util/log" +import { LiteLLMModels } from "./models" + +const log = Log.create({ service: "plugin.litellm" }) + +export async function LiteLLMPlugin(input: PluginInput): Promise { + return { + provider: { + id: "litellm", + async models(provider, ctx) { + const baseURL = (() => { + if (provider.options?.baseURL) return provider.options.baseURL as string + if (process.env["LITELLM_BASE_URL"]) return process.env["LITELLM_BASE_URL"] + return undefined + })() + + if (!baseURL) { + return provider.models + } + + const headers: Record = {} + if (ctx.auth?.type === "api" && ctx.auth.key) { + headers["Authorization"] = `Bearer ${ctx.auth.key}` + } else if (process.env["LITELLM_API_KEY"]) { + headers["Authorization"] = `Bearer ${process.env["LITELLM_API_KEY"]}` + } + + return LiteLLMModels.get(baseURL, headers, provider.models).catch((error) => { + log.error("failed to fetch litellm models", { error }) + return provider.models + }) + }, + }, + auth: { + provider: "litellm", + async loader(getAuth) { + const auth = await getAuth() + if (auth.type !== "api") { + // Fall back to env var + const envKey = process.env["LITELLM_API_KEY"] + if (envKey) return { apiKey: envKey } + return {} + } + + return { + apiKey: auth.key, + } + }, + methods: [ + { + label: "Enter LiteLLM API Key", + type: "api", + prompts: [ + { + type: "text", + key: "base_url", + message: "LiteLLM proxy base URL (e.g. http://localhost:4000)", + placeholder: "http://localhost:4000", + }, + ], + }, + ], + }, + } +} diff --git a/packages/opencode/src/plugin/litellm/models.ts b/packages/opencode/src/plugin/litellm/models.ts new file mode 100644 index 000000000000..24606c37bc7d --- /dev/null +++ b/packages/opencode/src/plugin/litellm/models.ts @@ -0,0 +1,133 @@ +import type { Model } from "@opencode-ai/sdk/v2" +import { InstallationVersion } from "@opencode-ai/core/installation/version" +import * as Log from "@opencode-ai/core/util/log" +import os from "os" + +const log = Log.create({ service: "plugin.litellm.models" }) + +interface LiteLLMModelEntry { + id: string + created?: number + owned_by?: string +} + +interface LiteLLMModelInfo { + max_tokens?: number + max_input_tokens?: number + max_output_tokens?: number + input_cost_per_token?: number + output_cost_per_token?: number + supports_vision?: boolean + supports_function_calling?: boolean + mode?: string +} + +async function fetchModelInfo( + baseURL: string, + headers: Record, +): Promise> { + try { + const url = `${baseURL.replace(/\/+$/, "")}/model/info` + const res = await fetch(url, { + headers, + signal: AbortSignal.timeout(5_000), + }) + if (!res.ok) return {} + const body = (await res.json()) as { data?: Array<{ model_name: string; model_info?: LiteLLMModelInfo }> } + if (!Array.isArray(body.data)) return {} + const info: Record = {} + for (const entry of body.data) { + if (entry.model_name && entry.model_info) { + info[entry.model_name] = entry.model_info + } + } + return info + } catch { + return {} + } +} + +function buildModel(id: string, baseURL: string, info?: LiteLLMModelInfo, prev?: Model): Model { + const maxContext = info?.max_tokens ?? info?.max_input_tokens ?? 128_000 + const maxOutput = info?.max_output_tokens ?? 16_384 + + const inputCost = info?.input_cost_per_token ?? 0 + const outputCost = info?.output_cost_per_token ?? 0 + + return { + id: id, + providerID: "litellm", + name: prev?.name ?? id, + api: { + id, + url: baseURL, + npm: "@ai-sdk/openai-compatible", + }, + status: "active", + headers: prev?.headers ?? {}, + options: prev?.options ?? {}, + cost: { + input: inputCost * 1_000_000, + output: outputCost * 1_000_000, + cache: { read: 0, write: 0 }, + }, + limit: { + context: maxContext, + output: maxOutput, + }, + capabilities: { + temperature: true, + reasoning: prev?.capabilities?.reasoning ?? false, + attachment: true, + toolcall: info?.supports_function_calling ?? prev?.capabilities?.toolcall ?? true, + input: { + text: true, + audio: false, + image: info?.supports_vision ?? prev?.capabilities?.input?.image ?? true, + video: false, + pdf: false, + }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + release_date: prev?.release_date ?? "", + variants: prev?.variants ?? {}, + } +} + +export async function get( + baseURL: string, + headers: Record = {}, + existing: Record = {}, +): Promise> { + const modelsURL = `${baseURL.replace(/\/+$/, "")}/v1/models` + const reqHeaders: Record = { + "User-Agent": `opencode/${InstallationVersion} litellm (${os.platform()} ${os.release()}; ${os.arch()})`, + ...headers, + } + + const res = await fetch(modelsURL, { + headers: reqHeaders, + signal: AbortSignal.timeout(5_000), + }) + if (!res.ok) throw new Error(`Failed to fetch models: ${res.status}`) + + const body = (await res.json()) as { data?: LiteLLMModelEntry[] } + if (!Array.isArray(body.data)) return existing + + // Enrich with per-model metadata from /model/info + const modelInfo = await fetchModelInfo(baseURL, reqHeaders) + + const result = { ...existing } + + for (const entry of body.data) { + const id = entry.id + if (!id) continue + const info = modelInfo[id] + result[id] = buildModel(id, baseURL, info, existing[id]) + } + + return result +} + +export * as LiteLLMModels from "./models" diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 8522432dff3a..e56c3de889db 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1263,6 +1263,17 @@ export const layer = Layer.effect( const providerID = ProviderV2.ID.make(p.id) if (disabled.has(providerID)) continue + // Seed provider entry if the plugin declares one but it's not in models.dev or config + if (!database[providerID]) { + database[providerID] = { + id: providerID, + name: providerID, + source: "env", + env: [], + options: {}, + models: {}, + } + } const provider = database[providerID] if (!provider) continue const pluginAuth = yield* auth.get(providerID).pipe(Effect.orDie) @@ -1280,6 +1291,11 @@ export const layer = Layer.effect( ]), ) }) + + // Ensure plugin-seeded providers appear in the state + if (!providers[providerID]) { + mergeProvider(providerID, { source: "env" }) + } } // extend database from config diff --git a/packages/opencode/test/plugin/litellm.test.ts b/packages/opencode/test/plugin/litellm.test.ts new file mode 100644 index 000000000000..6c40c338fff8 --- /dev/null +++ b/packages/opencode/test/plugin/litellm.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, test } from "bun:test" +import { LiteLLMPlugin } from "../../src/plugin/litellm/litellm" +import { LiteLLMModels } from "../../src/plugin/litellm/models" + +function makeServer(handler: (request: Request, url: URL) => Response | Promise) { + return Bun.serve({ + port: 0, + fetch: (request) => handler(request, new URL(request.url)), + }) +} + +describe("plugin.litellm", () => { + describe("LiteLLMModels.get", () => { + test("fetches models from /v1/models and returns model records", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ + data: [ + { id: "gpt-4o", created: 1700000000, owned_by: "openai" }, + { id: "claude-3-opus", created: 1700000000, owned_by: "anthropic" }, + ], + }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const models = await LiteLLMModels.get(server.url.toString(), {}) + expect(Object.keys(models)).toEqual(["gpt-4o", "claude-3-opus"]) + expect(models["gpt-4o"].providerID).toBe("litellm") + expect(models["gpt-4o"].api.npm).toBe("@ai-sdk/openai-compatible") + expect(models["claude-3-opus"].providerID).toBe("litellm") + }) + + test("enriches models with /model/info metadata", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ + data: [{ id: "gpt-4o" }], + }) + } + if (url.pathname === "/model/info") { + return Response.json({ + data: [ + { + model_name: "gpt-4o", + model_info: { + max_tokens: 128000, + max_output_tokens: 16384, + input_cost_per_token: 0.000005, + output_cost_per_token: 0.000015, + supports_vision: true, + supports_function_calling: true, + }, + }, + ], + }) + } + return new Response("not found", { status: 404 }) + }) + + const models = await LiteLLMModels.get(server.url.toString(), {}) + const model = models["gpt-4o"] + expect(model.limit.context).toBe(128000) + expect(model.limit.output).toBe(16384) + expect(model.cost.input).toBe(5) + expect(model.cost.output).toBe(15) + expect(model.capabilities.input.image).toBe(true) + expect(model.capabilities.toolcall).toBe(true) + }) + + test("uses defaults when /model/info has no entry for a model", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "unknown-model" }] }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const models = await LiteLLMModels.get(server.url.toString(), {}) + expect(models["unknown-model"].limit.context).toBe(128_000) + expect(models["unknown-model"].limit.output).toBe(16_384) + expect(models["unknown-model"].cost.input).toBe(0) + expect(models["unknown-model"].cost.output).toBe(0) + }) + + test("preserves existing model data when merging", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "gpt-4o" }] }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const existing = { + "gpt-4o": { + id: "gpt-4o", + providerID: "litellm", + name: "GPT-4o Custom Name", + capabilities: { reasoning: true }, + release_date: "2024-05-13", + } as any, + } + + const models = await LiteLLMModels.get(server.url.toString(), {}, existing) + expect(models["gpt-4o"].name).toBe("GPT-4o Custom Name") + expect(models["gpt-4o"].capabilities.reasoning).toBe(true) + expect(models["gpt-4o"].release_date).toBe("2024-05-13") + }) + + test("skips entries with empty id", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ + data: [{ id: "" }, { id: "valid-model" }, { id: undefined }], + }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const models = await LiteLLMModels.get(server.url.toString(), {}) + expect(Object.keys(models)).toEqual(["valid-model"]) + }) + + test("throws on non-ok /v1/models response", async () => { + using server = makeServer(() => new Response("unauthorized", { status: 401 })) + await expect(LiteLLMModels.get(server.url.toString(), {})).rejects.toThrow(/401/) + }) + + test("returns existing models when /v1/models returns no data array", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ object: "list" }) + } + return new Response("not found", { status: 404 }) + }) + + const existing = { "old-model": { id: "old-model" } as any } + const models = await LiteLLMModels.get(server.url.toString(), {}, existing) + expect(models).toEqual(existing) + }) + + test("gracefully handles /model/info failure", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "gpt-4o" }] }) + } + if (url.pathname === "/model/info") { + return new Response("server error", { status: 500 }) + } + return new Response("not found", { status: 404 }) + }) + + const models = await LiteLLMModels.get(server.url.toString(), {}) + expect(models["gpt-4o"]).toBeDefined() + expect(models["gpt-4o"].cost.input).toBe(0) + }) + + test("forwards authorization headers", async () => { + const capturedHeaders: Headers[] = [] + using server = makeServer((request, url) => { + capturedHeaders.push(request.headers) + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "m1" }] }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + await LiteLLMModels.get(server.url.toString(), { Authorization: "Bearer sk-test" }) + expect(capturedHeaders[0].get("authorization")).toBe("Bearer sk-test") + expect(capturedHeaders[0].get("user-agent")).toMatch(/^opencode\//) + }) + }) + + describe("LiteLLMPlugin", () => { + test("returns provider and auth hooks", async () => { + const hooks = await LiteLLMPlugin({} as any) + expect(hooks.provider).toBeDefined() + expect(hooks.provider!.id).toBe("litellm") + expect(typeof hooks.provider!.models).toBe("function") + expect(hooks.auth).toBeDefined() + expect(hooks.auth!.provider).toBe("litellm") + expect(typeof hooks.auth!.loader).toBe("function") + expect(hooks.auth!.methods).toHaveLength(1) + expect(hooks.auth!.methods[0].type).toBe("api") + }) + + test("auth loader returns env key when auth type is not api", async () => { + const original = process.env["LITELLM_API_KEY"] + process.env["LITELLM_API_KEY"] = "env-key-123" + try { + const hooks = await LiteLLMPlugin({} as any) + const result = await hooks.auth!.loader!(async () => ({ type: "oauth" }) as any, {} as any) + expect(result).toEqual({ apiKey: "env-key-123" }) + } finally { + if (original === undefined) delete process.env["LITELLM_API_KEY"] + else process.env["LITELLM_API_KEY"] = original + } + }) + + test("auth loader returns api key from stored auth", async () => { + const hooks = await LiteLLMPlugin({} as any) + const result = await hooks.auth!.loader!( + async () => ({ type: "api", key: "stored-key" }), + {} as any, + ) + expect(result).toEqual({ apiKey: "stored-key" }) + }) + + test("auth loader returns empty when no key available", async () => { + const original = process.env["LITELLM_API_KEY"] + delete process.env["LITELLM_API_KEY"] + try { + const hooks = await LiteLLMPlugin({} as any) + const result = await hooks.auth!.loader!(async () => ({ type: "oauth" }) as any, {} as any) + expect(result).toEqual({}) + } finally { + if (original !== undefined) process.env["LITELLM_API_KEY"] = original + } + }) + + test("provider.models returns existing models when no baseURL configured", async () => { + const original = process.env["LITELLM_BASE_URL"] + delete process.env["LITELLM_BASE_URL"] + try { + const hooks = await LiteLLMPlugin({} as any) + const existing = { "m1": { id: "m1" } as any } + const result = await hooks.provider!.models!({ models: existing, options: {} } as any, {} as any) + expect(result).toBe(existing) + } finally { + if (original !== undefined) process.env["LITELLM_BASE_URL"] = original + } + }) + + test("provider.models fetches from proxy when baseURL is set", async () => { + using server = makeServer((_, url) => { + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "proxy-model" }] }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const hooks = await LiteLLMPlugin({} as any) + const result = await hooks.provider!.models!( + { models: {}, options: { baseURL: server.url.toString() } } as any, + {} as any, + ) + expect(result["proxy-model"]).toBeDefined() + expect(result["proxy-model"].providerID).toBe("litellm") + }) + + test("provider.models falls back to existing on fetch error", async () => { + using server = makeServer(() => new Response("down", { status: 503 })) + + const hooks = await LiteLLMPlugin({} as any) + const existing = { "fallback": { id: "fallback" } as any } + const result = await hooks.provider!.models!( + { models: existing, options: { baseURL: server.url.toString() } } as any, + {} as any, + ) + expect(result).toBe(existing) + }) + + test("provider.models uses auth context for authorization header", async () => { + const capturedHeaders: Headers[] = [] + using server = makeServer((request, url) => { + capturedHeaders.push(request.headers) + if (url.pathname === "/v1/models") { + return Response.json({ data: [{ id: "m1" }] }) + } + if (url.pathname === "/model/info") { + return Response.json({ data: [] }) + } + return new Response("not found", { status: 404 }) + }) + + const hooks = await LiteLLMPlugin({} as any) + await hooks.provider!.models!( + { models: {}, options: { baseURL: server.url.toString() } } as any, + { auth: { type: "api", key: "ctx-key" } } as any, + ) + expect(capturedHeaders[0].get("authorization")).toBe("Bearer ctx-key") + }) + }) +}) diff --git a/packages/ui/src/components/provider-icons/sprite.svg b/packages/ui/src/components/provider-icons/sprite.svg index 68b99ce56d4a..cc948dc66b40 100644 --- a/packages/ui/src/components/provider-icons/sprite.svg +++ b/packages/ui/src/components/provider-icons/sprite.svg @@ -1131,5 +1131,8 @@ d="M 554.228 729.711 C 659.104 725.931 747.227 807.802 751.164 912.672 C 755.1 1017.54 673.362 1105.79 568.498 1109.88 C 463.411 1113.98 374.937 1032.03 370.992 926.942 C 367.047 821.849 449.129 733.498 554.228 729.711 z" > + + 🚅 + diff --git a/packages/ui/src/components/provider-icons/types.ts b/packages/ui/src/components/provider-icons/types.ts index 1c6f5fe6d2f0..181da3b1a622 100644 --- a/packages/ui/src/components/provider-icons/types.ts +++ b/packages/ui/src/components/provider-icons/types.ts @@ -33,6 +33,7 @@ export const iconNames = [ "ovhcloud", "openrouter", "llmgateway", + "litellm", "opencode", "opencode-go", "openai",