From 79b1ec2530c0cdaf6d3f29f1ce960819403c8301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Ho=C3=A0i=20Nam?= Date: Tue, 19 May 2026 21:49:21 +0700 Subject: [PATCH] feat(ai): add LiteLLM adapter --- packages/ai/litellm/package.json | 25 +++++++++ packages/ai/litellm/src/index.test.ts | 80 +++++++++++++++++++++++++++ packages/ai/litellm/src/index.ts | 76 +++++++++++++++++++++++++ packages/ai/litellm/tsconfig.json | 10 ++++ pnpm-lock.yaml | 6 ++ 5 files changed, 197 insertions(+) create mode 100644 packages/ai/litellm/package.json create mode 100644 packages/ai/litellm/src/index.test.ts create mode 100644 packages/ai/litellm/src/index.ts create mode 100644 packages/ai/litellm/tsconfig.json diff --git a/packages/ai/litellm/package.json b/packages/ai/litellm/package.json new file mode 100644 index 00000000..24ee4a7a --- /dev/null +++ b/packages/ai/litellm/package.json @@ -0,0 +1,25 @@ +{ + "name": "@profullstack/sh1pt-ai-litellm", + "version": "0.1.15", + "type": "module", + "main": "./src/index.ts", + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "prepublishOnly": "pnpm build" + }, + "dependencies": { + "@profullstack/sh1pt-core": "workspace:*" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/profullstack/sh1pt.git", + "directory": "packages/ai/litellm" + }, + "homepage": "https://sh1pt.com", + "bugs": "https://github.com/profullstack/sh1pt/issues", + "files": [ + "dist" + ] +} diff --git a/packages/ai/litellm/src/index.test.ts b/packages/ai/litellm/src/index.test.ts new file mode 100644 index 00000000..c420f766 --- /dev/null +++ b/packages/ai/litellm/src/index.test.ts @@ -0,0 +1,80 @@ +import { smokeTest } from '@profullstack/sh1pt-core/testing'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import adapter from './index.js'; + +smokeTest(adapter, { idPrefix: 'ai' }); + +const ctx = (secrets: Record = { LITELLM_API_KEY: 'test-key' }, dryRun = false) => ({ + secret: (key: string) => secrets[key], + log: () => {}, + dryRun, +}); + +describe('LiteLLM OpenAI-compatible generation', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('short-circuits dry-run before network calls', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + const result = await adapter.generate(ctx({ LITELLM_API_KEY: 'test-key' }, true), 'hello', {}, {}); + + expect(result).toEqual({ text: '[dry-run]', model: 'gpt-4o-mini' }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('posts chat completions requests and maps usage tokens', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + choices: [{ message: { content: 'hi from litellm' } }], + model: 'anthropic/claude-3-5-sonnet', + usage: { prompt_tokens: 11, completion_tokens: 5 }, + }), + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await adapter.generate(ctx(), 'hello', { + model: 'anthropic/claude-3-5-sonnet', + system: 'be brief', + maxTokens: 20, + temperature: 0.2, + extra: { top_p: 0.9 }, + }, { baseUrl: 'https://proxy.example.com/' }); + + expect(fetchMock).toHaveBeenCalledOnce(); + const call = fetchMock.mock.calls[0]; + expect(call).toBeDefined(); + const [url, request] = call!; + expect(url).toBe('https://proxy.example.com/v1/chat/completions'); + expect(request.headers.authorization).toBe('Bearer test-key'); + expect(JSON.parse(request.body)).toEqual({ + model: 'anthropic/claude-3-5-sonnet', + messages: [ + { role: 'system', content: 'be brief' }, + { role: 'user', content: 'hello' }, + ], + max_tokens: 20, + temperature: 0.2, + top_p: 0.9, + }); + expect(result).toEqual({ + text: 'hi from litellm', + model: 'anthropic/claude-3-5-sonnet', + inputTokens: 11, + outputTokens: 5, + }); + }); + + it('includes status and response body excerpt on errors', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: false, + status: 401, + text: async () => 'invalid proxy key'.repeat(30), + })); + + await expect(adapter.generate(ctx(), 'hello', {}, {})).rejects.toThrow(/LiteLLM 401: invalid proxy key/); + }); +}); diff --git a/packages/ai/litellm/src/index.ts b/packages/ai/litellm/src/index.ts new file mode 100644 index 00000000..05b22f83 --- /dev/null +++ b/packages/ai/litellm/src/index.ts @@ -0,0 +1,76 @@ +import { defineAi, tokenSetup } from '@profullstack/sh1pt-core'; + +interface Config { + baseUrl?: string; +} + +const DEFAULT_BASE = 'http://localhost:4000'; + +const trimTrailingSlash = (value: string) => value.replace(/\/+$/, ''); + +export default defineAi({ + id: 'ai-litellm', + label: 'LiteLLM', + defaultModel: 'gpt-4o-mini', + models: [ + 'gpt-4o-mini', + 'gpt-4o', + 'anthropic/claude-3-5-sonnet', + 'gemini/gemini-1.5-pro', + 'ollama/llama3.1', + ], + + async generate(ctx, prompt, opts, config) { + const apiKey = ctx.secret('LITELLM_API_KEY'); + if (!apiKey) throw new Error('LITELLM_API_KEY not in vault'); + const model = opts.model ?? 'gpt-4o-mini'; + ctx.log(`litellm · model=${model} · ${prompt.length} chars in`); + if (ctx.dryRun) return { text: '[dry-run]', model }; + + const messages: Array<{ role: string; content: string }> = []; + if (opts.system) messages.push({ role: 'system', content: opts.system }); + messages.push({ role: 'user', content: prompt }); + + const baseUrl = trimTrailingSlash(config.baseUrl ?? DEFAULT_BASE); + const res = await fetch(`${baseUrl}/v1/chat/completions`, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model, + messages, + ...(opts.maxTokens !== undefined ? { max_tokens: opts.maxTokens } : {}), + ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}), + ...opts.extra, + }), + }); + if (!res.ok) throw new Error(`LiteLLM ${res.status}: ${(await res.text()).slice(0, 200)}`); + const data = (await res.json()) as { + choices: Array<{ message?: { content?: string } }>; + model: string; + usage?: { prompt_tokens?: number; completion_tokens?: number }; + }; + return { + text: data.choices[0]?.message?.content ?? '', + model: data.model, + inputTokens: data.usage?.prompt_tokens, + outputTokens: data.usage?.completion_tokens, + }; + }, + + setup: tokenSetup({ + secretKey: 'LITELLM_API_KEY', + label: 'LiteLLM', + vendorDocUrl: 'https://docs.litellm.ai/docs/proxy/quick_start', + steps: [ + 'Start a LiteLLM proxy or use a hosted LiteLLM gateway', + 'Create or copy the proxy API key', + 'Paste below; sh1pt encrypts it in the vault', + ], + fields: [ + { key: 'baseUrl', message: 'LiteLLM proxy base URL (default: http://localhost:4000):' }, + ], + }), +}); diff --git a/packages/ai/litellm/tsconfig.json b/packages/ai/litellm/tsconfig.json new file mode 100644 index 00000000..abb4fc01 --- /dev/null +++ b/packages/ai/litellm/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": [ + "src/**/*" + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23f068a5..6a156c9f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -371,6 +371,12 @@ importers: specifier: workspace:* version: link:../../core + packages/ai/litellm: + dependencies: + '@profullstack/sh1pt-core': + specifier: workspace:* + version: link:../../core + packages/ai/mancer: dependencies: '@profullstack/sh1pt-core':