diff --git a/plugins/openai/src/realtime/api_proto.ts b/plugins/openai/src/realtime/api_proto.ts index ed07a34e6..75e66c3e6 100644 --- a/plugins/openai/src/realtime/api_proto.ts +++ b/plugins/openai/src/realtime/api_proto.ts @@ -91,17 +91,26 @@ export interface Tool { }; } -export type TurnDetectionType = { - type: 'server_vad'; - threshold?: number; // 0.0 to 1.0, default: 0.5 - prefix_padding_ms?: number; // default: 300 - silence_duration_ms?: number; // default: 200 - create_response?: boolean; // default: true - interrupt_response?: boolean; // default: true -}; +export type TurnDetectionType = + | { + type: 'semantic_vad'; + eagerness?: 'auto' | 'low' | 'medium' | 'high'; // default: auto + create_response?: boolean; // default: true + interrupt_response?: boolean; // default: true + } + | { + type: 'server_vad'; + threshold?: number; // 0.0 to 1.0, default: 0.5 + prefix_padding_ms?: number; // default: 300 + silence_duration_ms?: number; // default: 200 + create_response?: boolean; // default: true + interrupt_response?: boolean; // default: true + }; export type InputAudioTranscription = { model: InputTranscriptionModel; + language?: string; + prompt?: string; }; export interface InputTextContent { diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 53a085bd9..a93bf9b6b 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -22,7 +22,6 @@ import { import { Mutex } from '@livekit/mutex'; import type { AudioResampler } from '@livekit/rtc-node'; import { AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; -import type { GenerationCreatedEvent } from 'agents/dist/llm/realtime.js'; import { type MessageEvent, WebSocket } from 'ws'; import * as api_proto from './api_proto.js'; @@ -1011,7 +1010,7 @@ export class RealtimeSession extends llm.RealtimeSession { messageStream: this.currentGeneration.messageChannel.stream(), functionStream: this.currentGeneration.functionChannel.stream(), userInitiated: false, - } as GenerationCreatedEvent; + } as llm.GenerationCreatedEvent; const clientEventId = event.response.metadata?.client_event_id; if (clientEventId) { @@ -1414,7 +1413,7 @@ export class RealtimeSession extends llm.RealtimeSession { messageStream: this.currentGeneration.messageChannel.stream(), functionStream: this.currentGeneration.functionChannel.stream(), userInitiated: false, - } as GenerationCreatedEvent; + } as llm.GenerationCreatedEvent; const handle = this.responseCreatedFutures[responseId]; if (handle) {