Skip to content

1 gemini api not supported #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataopsnick merged 2 commits into from
Jun 1, 2025
Merged

1 gemini api not supported #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c55bb56
fix package.json build config
Jun 1, 2025
17f52b4
Bug fix openai proxy interface
Jun 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,51 @@ for await (const chunk of chunks) {

```

### Python

Works in Google Colab

```Python
from anyio import run
from openai import AsyncOpenAI

# Configuration
server = 'gemini'
base_url = f'https://abcdefghijklmnopqrstuvwxyz.lambda-url.us-west-2.on.aws/{server}/v1/'
api_key = 'GEMINI_API_KEY' # Use the same token as in your YAML

# Create OpenAI client
openai = AsyncOpenAI(
base_url=base_url,
api_key=api_key,
)

async def main():
model = 'gemini-2.0-flash'
prompt = 'Tell me a joke.'

params = {
'model': model,
'messages': [{'role': 'user', 'content': prompt}],
'stream': True,
}

# Create streaming chat completion
chunks = await openai.chat.completions.create(**params)
response = ''

# Iterate through streaming chunks
async for chunk in chunks:
if chunk.choices and chunk.choices[0].delta.content:
content = chunk.choices[0].delta.content
response += content
print(content, end='', flush=True)

print(f"\n\nFull response: {response}")

# Run the async function
await main()```

### Test

```
Expand Down
110 changes: 92 additions & 18 deletions src/llm_client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,27 @@
import OpenAi from 'openai';
import { OpenAiSettings } from './openai_settings';
import { GeminiSettings } from './gemini_settings';
import { GoogleGenerativeAI, GenerativeModel } from "@google/genai"; // esbuild will mark this as external
import { GoogleGenerativeAI, GenerativeModel } from "@google/generative-ai";
import { ChatCompletion, ChatCompletionCreateParams } from 'openai/resources/chat/completions';
import { ReadableStream as PolyfillReadableStream } from 'web-streams-polyfill'; // Corrected path
import { ReadableStream as PolyfillReadableStream } from 'web-streams-polyfill';
import { TextEncoder, TextDecoder } from 'util';

// Use native ReadableStream if available, otherwise use the polyfill
const ReadableStream = globalThis.ReadableStream || PolyfillReadableStream;

const getPartialKey = (key: string | undefined | null): string => {
if (!key || key.length < 8) {
return key || "undefined/empty";
}
return `${key.substring(0, 5)}...${key.substring(key.length - 3)}`;
};

export class LlmClient {
public openai: OpenAi | null;
public gemini: GoogleGenerativeAI | null;
public model: string;
public openaiSettings: OpenAiSettings | null;
public geminiSettings: GeminiSettings | null;


constructor(settings: OpenAiSettings | GeminiSettings) {
if ('url' in settings) {
// It's OpenAiSettings
Expand All @@ -31,7 +36,56 @@ export class LlmClient {
// It's GeminiSettings
this.geminiSettings = settings;
this.openaiSettings = null;
this.gemini = new GoogleGenerativeAI({ apiKey: settings.token });

// EXTENSIVE DEBUG LOGGING
const envApiKey = process.env.GOOGLE_API_KEY;
const yamlApiKey = settings.token;

console.log(`[DEBUG_ENV] All environment variables:`, Object.keys(process.env).filter(k => k.includes('API') || k.includes('KEY') || k.includes('GOOGLE')));
console.log(`[DEBUG_ENV] GOOGLE_API_KEY exists:`, envApiKey !== undefined);
console.log(`[DEBUG_ENV] GOOGLE_API_KEY value:`, getPartialKey(envApiKey));
console.log(`[DEBUG_YAML] YAML token value:`, getPartialKey(yamlApiKey));

// Use YAML config by default (since local works with YAML)
const effectiveApiKey = yamlApiKey;

console.log(`[DEBUG_EFFECTIVE] Using API key:`, getPartialKey(effectiveApiKey));
console.log(`[DEBUG_EFFECTIVE] Key length:`, effectiveApiKey?.length);
console.log(`[DEBUG_EFFECTIVE] Key starts with AIza:`, effectiveApiKey?.startsWith('AIza'));

if (!effectiveApiKey || effectiveApiKey.trim() === "") {
throw new Error("Gemini API key is missing");
}

// Log the exact constructor call
console.log(`[DEBUG_CONSTRUCTOR] About to call GoogleGenerativeAI constructor`);
console.log(`[DEBUG_CONSTRUCTOR] Constructor argument type:`, typeof effectiveApiKey);

try {
// Try different ways of passing the API key
console.log(`[DEBUG_CONSTRUCTOR] Method 1: Direct string`);
this.gemini = new GoogleGenerativeAI(effectiveApiKey);
console.log(`[DEBUG_CONSTRUCTOR] GoogleGenerativeAI created successfully`);

// Test the client immediately
console.log(`[DEBUG_TEST] Testing client initialization...`);
const testModel = this.gemini.getGenerativeModel({ model: 'gemini-2.0-flash' });
console.log(`[DEBUG_TEST] Model instance created successfully`);

} catch (initError: any) {
console.error(`[DEBUG_ERROR] GoogleGenerativeAI constructor failed:`, initError.message);

// Try alternative constructor format
console.log(`[DEBUG_CONSTRUCTOR] Method 2: Object format`);
try {
this.gemini = new GoogleGenerativeAI({ apiKey: effectiveApiKey });
console.log(`[DEBUG_CONSTRUCTOR] GoogleGenerativeAI created with object format`);
} catch (altError: any) {
console.error(`[DEBUG_ERROR] Alternative constructor also failed:`, altError.message);
throw initError;
}
}

this.openai = null;
}

Expand All @@ -55,14 +109,28 @@ export class LlmClient {
const resp = await this.openai.chat.completions.create(params);
return resp.toReadableStream() as any;
} else if (this.gemini) {
const model: GenerativeModel = this.gemini.getGenerativeModel({ model: this.model });
const chat = model.startChat({
history: [], // IMPORTANT: Add history here later
});

const result = await chat.sendMessageStream({ message: content }); // Mimic example

return result.stream as any;
console.log(`[DEBUG_STREAM] Starting streaming with model: ${this.model}`);
console.log(`[DEBUG_STREAM] Content: ${content.substring(0, 50)}...`);

try {
const model: GenerativeModel = this.gemini.getGenerativeModel({ model: this.model });
console.log(`[DEBUG_STREAM] Model instance obtained`);

const chat = model.startChat({ history: [] });
console.log(`[DEBUG_STREAM] Chat started`);

console.log(`[DEBUG_STREAM] About to call sendMessageStream...`);
const result = await chat.sendMessageStream([{ text: content }]);
console.log(`[DEBUG_STREAM] sendMessageStream succeeded`);

return result.stream as any;
} catch (e: any) {
console.error(`[DEBUG_STREAM_ERROR] Stream error:`, e.message);
console.error(`[DEBUG_STREAM_ERROR] Error details:`, JSON.stringify(e.errorDetails || {}, null, 2));
console.error(`[DEBUG_STREAM_ERROR] Error cause:`, JSON.stringify(e.cause || {}, null, 2));
console.error(`[DEBUG_STREAM_ERROR] Full error:`, e);
throw e;
}
} else {
throw new Error(`Unsupported provider`);
}
Expand Down Expand Up @@ -92,12 +160,18 @@ export class LlmClient {
}
}

async chatCompletionNonStreaming(content: string): Promise<ChatCompletion> {
async chatCompletionNonStreaming(content: string): Promise<ChatCompletion | string> {
if (this.openai) {
const params = this.completionParams(this.model, content);
return await this.openai.chat.completions.create(params) as ChatCompletion;
const params = this.completionParams(this.model, content);
return await this.openai.chat.completions.create(params) as ChatCompletion;
}
if (this.gemini) {
const model: GenerativeModel = this.gemini.getGenerativeModel({ model: this.model });
const chat = model.startChat({ history: [] });
const result = await chat.sendMessage([{ text: content }]);
return result.response.text();
}
throw new Error("Not implemented");
throw new Error("Not implemented for this provider for non-streaming or client not initialized.");
}

async createCompletionNonStreaming(params: OpenAi.Chat.Completions.ChatCompletionCreateParamsNonStreaming): Promise<OpenAi.Chat.Completions.ChatCompletion> {
Expand All @@ -108,4 +182,4 @@ export class LlmClient {
}
return response;
}
}
}
Loading