From ff269c7d8d9eb0584de2a10eb01ef8a71e565778 Mon Sep 17 00:00:00 2001 From: Yaniv Perez Date: Tue, 16 Sep 2025 03:08:35 +0700 Subject: [PATCH 1/4] Update realtime_model.ts types --- plugins/openai/src/realtime/realtime_model.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 53a085bd9..a93bf9b6b 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -22,7 +22,6 @@ import { import { Mutex } from '@livekit/mutex'; import type { AudioResampler } from '@livekit/rtc-node'; import { AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; -import type { GenerationCreatedEvent } from 'agents/dist/llm/realtime.js'; import { type MessageEvent, WebSocket } from 'ws'; import * as api_proto from './api_proto.js'; @@ -1011,7 +1010,7 @@ export class RealtimeSession extends llm.RealtimeSession { messageStream: this.currentGeneration.messageChannel.stream(), functionStream: this.currentGeneration.functionChannel.stream(), userInitiated: false, - } as GenerationCreatedEvent; + } as llm.GenerationCreatedEvent; const clientEventId = event.response.metadata?.client_event_id; if (clientEventId) { @@ -1414,7 +1413,7 @@ export class RealtimeSession extends llm.RealtimeSession { messageStream: this.currentGeneration.messageChannel.stream(), functionStream: this.currentGeneration.functionChannel.stream(), userInitiated: false, - } as GenerationCreatedEvent; + } as llm.GenerationCreatedEvent; const handle = this.responseCreatedFutures[responseId]; if (handle) { From f65226c4ced8e511b3e4b76bcc3d4eae2710832a Mon Sep 17 00:00:00 2001 From: Yaniv Perez Date: Tue, 16 Sep 2025 12:13:37 +0700 Subject: [PATCH 2/4] Update api_proto.ts https://learn.microsoft.com/en-us/azure/ai-foundry/openai/realtime-audio-reference#realtimeturndetection --- plugins/openai/src/realtime/api_proto.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/openai/src/realtime/api_proto.ts b/plugins/openai/src/realtime/api_proto.ts index ed07a34e6..791ee5359 100644 --- a/plugins/openai/src/realtime/api_proto.ts +++ b/plugins/openai/src/realtime/api_proto.ts @@ -92,13 +92,18 @@ export interface Tool { } export type TurnDetectionType = { + type: 'semantic_vad'; + eagerness?: 'auto' | 'low' | 'medium' | 'high'; // default: auto + create_response?: boolean; // default: true + interrupt_response?: boolean; // default: true +} | { type: 'server_vad'; threshold?: number; // 0.0 to 1.0, default: 0.5 prefix_padding_ms?: number; // default: 300 silence_duration_ms?: number; // default: 200 create_response?: boolean; // default: true interrupt_response?: boolean; // default: true -}; +};; export type InputAudioTranscription = { model: InputTranscriptionModel; From 5b5ded2835c2da94de3b9a2dafd93b713ceb9729 Mon Sep 17 00:00:00 2001 From: Yaniv Perez Date: Tue, 16 Sep 2025 12:17:16 +0700 Subject: [PATCH 3/4] Update api_proto.ts https://learn.microsoft.com/en-us/azure/ai-foundry/openai/realtime-audio-reference#realtimeaudioinputtranscriptionsettings --- plugins/openai/src/realtime/api_proto.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/openai/src/realtime/api_proto.ts b/plugins/openai/src/realtime/api_proto.ts index 791ee5359..b515294b8 100644 --- a/plugins/openai/src/realtime/api_proto.ts +++ b/plugins/openai/src/realtime/api_proto.ts @@ -107,6 +107,8 @@ export type TurnDetectionType = { export type InputAudioTranscription = { model: InputTranscriptionModel; + language?: string; + prompt?: string; }; export interface InputTextContent { From ec90edddecde41e464e38f4f95d13f183fd8d297 Mon Sep 17 00:00:00 2001 From: Yaniv Perez Date: Tue, 16 Sep 2025 12:24:30 +0700 Subject: [PATCH 4/4] prettier https://learn.microsoft.com/en-us/azure/ai-foundry/openai/realtime-audio-reference#realtimeaudioinputtranscriptionsettings --- plugins/openai/src/realtime/api_proto.ts | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/plugins/openai/src/realtime/api_proto.ts b/plugins/openai/src/realtime/api_proto.ts index b515294b8..75e66c3e6 100644 --- a/plugins/openai/src/realtime/api_proto.ts +++ b/plugins/openai/src/realtime/api_proto.ts @@ -91,19 +91,21 @@ export interface Tool { }; } -export type TurnDetectionType = { - type: 'semantic_vad'; - eagerness?: 'auto' | 'low' | 'medium' | 'high'; // default: auto - create_response?: boolean; // default: true - interrupt_response?: boolean; // default: true -} | { - type: 'server_vad'; - threshold?: number; // 0.0 to 1.0, default: 0.5 - prefix_padding_ms?: number; // default: 300 - silence_duration_ms?: number; // default: 200 - create_response?: boolean; // default: true - interrupt_response?: boolean; // default: true -};; +export type TurnDetectionType = + | { + type: 'semantic_vad'; + eagerness?: 'auto' | 'low' | 'medium' | 'high'; // default: auto + create_response?: boolean; // default: true + interrupt_response?: boolean; // default: true + } + | { + type: 'server_vad'; + threshold?: number; // 0.0 to 1.0, default: 0.5 + prefix_padding_ms?: number; // default: 300 + silence_duration_ms?: number; // default: 200 + create_response?: boolean; // default: true + interrupt_response?: boolean; // default: true + }; export type InputAudioTranscription = { model: InputTranscriptionModel;