diff --git a/agents/src/inference/api_protos.ts b/agents/src/inference/api_protos.ts index acb069e8a..f8b7a7bb8 100644 --- a/agents/src/inference/api_protos.ts +++ b/agents/src/inference/api_protos.ts @@ -14,9 +14,17 @@ export const ttsSessionCreateEventSchema = z.object({ transcript: z.string().optional(), }); +export const ttsGenerationConfigSchema = z.object({ + voice: z.string().optional(), + language: z.string().optional(), + model: z.string().optional(), +}); + export const ttsInputTranscriptEventSchema = z.object({ type: z.literal('input_transcript'), transcript: z.string(), + generation_config: ttsGenerationConfigSchema.optional(), + extra: z.record(z.string(), z.unknown()).optional(), }); export const ttsSessionFlushEventSchema = z.object({ @@ -109,6 +117,7 @@ export const ttsServerEventSchema = z.union([ ttsUnknownServerEventSchema, ]); +export type TtsGenerationConfig = z.infer; export type TtsSessionCreateEvent = z.infer; export type TtsInputTranscriptEvent = z.infer; export type TtsSessionFlushEvent = z.infer; diff --git a/agents/src/inference/index.ts b/agents/src/inference/index.ts index b6847f59c..e77981c79 100644 --- a/agents/src/inference/index.ts +++ b/agents/src/inference/index.ts @@ -14,12 +14,17 @@ export { type InferenceLLMOptions, type LLMModels, type XAIModels, + type ZAIModels, } from './llm.js'; export { normalizeSTTFallback, parseSTTModelString, STT, + type DeepgramFluxModels, + type DeepgramFluxOptions, + type InworldSTTModels, + type InworldSTTOptions, type STTFallbackModel, type STTFallbackModelType, type STTLanguages, @@ -39,6 +44,8 @@ export { type TTSModels, type ModelWithVoice as TTSModelString, type TTSOptions, + type XaiTTSModels, + type XaiTTSOptions, } from './tts.js'; export { llm, stt, tts }; diff --git a/agents/src/inference/llm.ts b/agents/src/inference/llm.ts index af11b70f7..7196ba998 100644 --- a/agents/src/inference/llm.ts +++ b/agents/src/inference/llm.ts @@ -17,7 +17,10 @@ import { } from './utils.js'; export type OpenAIModels = + | 'openai/gpt-5.5' | 'openai/gpt-5.4' + | 'openai/gpt-5.4-mini' + | 'openai/gpt-5.4-nano' | 'openai/gpt-5.3-chat-latest' | 'openai/gpt-5.2' | 'openai/gpt-5.2-chat-latest' @@ -31,21 +34,24 @@ export type OpenAIModels = | 'openai/gpt-4.1-nano' | 'openai/gpt-4o' | 'openai/gpt-4o-mini' + | 'openai/chat-latest' | 'openai/gpt-oss-120b'; export type GoogleModels = - | 'google/gemini-3-pro' + | 'google/gemini-3.1-pro' | 'google/gemini-3-flash' + | 'google/gemini-3.1-flash-lite' + | 'google/gemini-3.5-flash' | 'google/gemini-2.5-pro' | 'google/gemini-2.5-flash' - | 'google/gemini-2.5-flash-lite' - | 'google/gemini-2.0-flash' - | 'google/gemini-2.0-flash-lite'; + | 'google/gemini-2.5-flash-lite'; -export type MoonshotModels = 'moonshotai/kimi-k2-instruct'; +export type MoonshotModels = 'moonshotai/kimi-k2.5' | 'moonshotai/kimi-k2.6'; export type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2'; +export type ZAIModels = 'zai/glm-5.1'; + export type XAIModels = | 'xai/grok-4-1-fast-non-reasoning' | 'xai/grok-4-1-fast-reasoning' @@ -57,6 +63,7 @@ type ChatCompletionPredictionContentParam = Expand; type WebSearchOptions = Expand; type ToolChoice = Expand; +type ResponseFormat = Expand; type Verbosity = 'low' | 'medium' | 'high'; export interface ChatCompletionOptions extends Record { @@ -88,8 +95,7 @@ export interface ChatCompletionOptions extends Record { // livekit-typed arguments tool_choice?: ToolChoice; - // TODO(brian): support response format - // response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format'] + response_format?: ResponseFormat; } export type LLMModels = @@ -97,6 +103,7 @@ export type LLMModels = | GoogleModels | MoonshotModels | DeepSeekModels + | ZAIModels | XAIModels | AnyString; @@ -275,7 +282,6 @@ export class LLM extends llm.LLM { parallelToolCalls, toolChoice, inferenceClass, - // TODO(AJS-270): Add response_format parameter support extraKwargs, }: { chatCtx: llm.ChatContext; @@ -284,7 +290,6 @@ export class LLM extends llm.LLM { parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; inferenceClass?: InferenceClass; - // TODO(AJS-270): Add responseFormat parameter extraKwargs?: Record; }): LLMStream { let modelOptions: Record = { ...(extraKwargs || {}) }; @@ -310,8 +315,6 @@ export class LLM extends llm.LLM { const resolvedInferenceClass = inferenceClass !== undefined ? inferenceClass : this.opts.inferenceClass; - // TODO(AJS-270): Add response_format support here - modelOptions = { ...modelOptions, ...this.opts.modelOptions }; return new LLMStream(this, { diff --git a/agents/src/inference/stt.ts b/agents/src/inference/stt.ts index 67a1094ab..2acb1bc80 100644 --- a/agents/src/inference/stt.ts +++ b/agents/src/inference/stt.ts @@ -29,7 +29,6 @@ import { import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js'; export type DeepgramModels = - | 'deepgram/flux-general' | 'deepgram/nova-3' | 'deepgram/nova-3-medical' | 'deepgram/nova-2' @@ -37,11 +36,17 @@ export type DeepgramModels = | 'deepgram/nova-2-conversationalai' | 'deepgram/nova-2-phonecall'; -export type CartesiaModels = 'cartesia/ink-whisper'; +export type DeepgramFluxModels = + | 'deepgram/flux-general' + | 'deepgram/flux-general-en' + | 'deepgram/flux-general-multi'; + +export type CartesiaModels = 'cartesia/ink-whisper' | 'cartesia/ink-2' | 'cartesia/ink-2-latest'; export type AssemblyaiModels = | 'assemblyai/universal-streaming' - | 'assemblyai/universal-streaming-multilingual'; + | 'assemblyai/universal-streaming-multilingual' + | 'assemblyai/u3-rt-pro'; export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime'; @@ -49,6 +54,8 @@ export type XaiSTTModels = 'xai/stt-1'; export type SpeechmaticsModels = 'speechmatics/enhanced' | 'speechmatics/standard'; +export type InworldSTTModels = 'inworld/inworld-stt-1'; + export interface CartesiaOptions { /** Minimum volume threshold. Default: not specified. */ min_volume?: number; @@ -79,8 +86,23 @@ export interface DeepgramOptions { mip_opt_out?: boolean; /** Enable speaker diarization. Default: false. */ diarize?: boolean; - /** Eager end-of-turn threshold (0.0–1.0). Enables preflight transcripts for preemptive generation. */ +} + +export interface DeepgramFluxOptions { + /** Eager end-of-turn threshold (0.3–0.9). Enables preflight transcripts for preemptive generation. Default: 0.5. */ eager_eot_threshold?: number; + /** End-of-turn threshold (0.5–0.9). */ + eot_threshold?: number; + /** End-of-turn timeout in milliseconds. */ + eot_timeout_ms?: number; + /** Key terms for recognition. */ + keyterm?: string | string[]; + /** Opt out of model improvement program. Default: false. */ + mip_opt_out?: boolean; + /** Language hint. */ + language_hint?: string; + /** Enable automatic language detection. */ + detect_language?: boolean; } export interface AssemblyAIOptions { @@ -148,6 +170,27 @@ export interface SpeechmaticsOptions { transcript_filtering_config?: Record; } +export interface InworldSTTOptions { + /** Enable Voice Profile detection. Default: true. */ + enable_voice_profile?: boolean; + /** Max labels per category in voice-profile responses (1–20). Default: 10. */ + voice_profile_top_n?: number; + /** Enable word-level timestamps. Default: true. */ + include_word_timestamps?: boolean; + /** Wire-format encoding sent to Inworld. Default: LINEAR16. */ + audio_encoding?: 'LINEAR16' | 'AUTO_DETECT'; + /** Stop transcription after this many seconds of silence; 0 disables. */ + inactivity_timeout_seconds?: number; + /** End-of-turn confidence threshold (0.0–1.0). Default: 0.5. */ + end_of_turn_confidence_threshold?: number; + /** Domain-specific contextual hints passed to the model. */ + prompts?: string[]; + /** Minimum end-of-turn silence in milliseconds when confident. */ + min_end_of_turn_silence_when_confident?: number; + /** VAD threshold (0.0–1.0). Default: 0.5. */ + vad_threshold?: number; +} + export type STTLanguages = | 'multi' | 'en' @@ -173,27 +216,33 @@ function diarizationEnabled(extraKwargs: Record | undefined): b type _STTModels = | DeepgramModels + | DeepgramFluxModels | CartesiaModels | AssemblyaiModels | ElevenlabsSTTModels | XaiSTTModels - | SpeechmaticsModels; + | SpeechmaticsModels + | InworldSTTModels; export type STTModels = _STTModels | 'auto' | AnyString; export type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels; -export type STTOptions = TModel extends DeepgramModels - ? DeepgramOptions - : TModel extends CartesiaModels - ? CartesiaOptions - : TModel extends AssemblyaiModels - ? AssemblyAIOptions - : TModel extends XaiSTTModels - ? XaiOptions - : TModel extends SpeechmaticsModels - ? SpeechmaticsOptions - : Record; +export type STTOptions = TModel extends DeepgramFluxModels + ? DeepgramFluxOptions + : TModel extends DeepgramModels + ? DeepgramOptions + : TModel extends CartesiaModels + ? CartesiaOptions + : TModel extends AssemblyaiModels + ? AssemblyAIOptions + : TModel extends XaiSTTModels + ? XaiOptions + : TModel extends SpeechmaticsModels + ? SpeechmaticsOptions + : TModel extends InworldSTTModels + ? InworldSTTOptions + : Record; /** Inference Fallback Adapter: configuration for a fallback STT model that runs server-side in LiveKit Inference, providing automatic fallback between providers. Extra fields are passed through to the provider. */ export interface STTFallbackModel { @@ -532,6 +581,7 @@ export class SpeechStream extends BaseSpeechStream { private reconnectEvent = new Event(); private stt: STT; private connOptions: APIConnectOptions; + private activeWs?: WebSocket; #logger = log(); @@ -563,7 +613,23 @@ export class SpeechStream extends BaseSpeechStream { language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language, modelOptions: mergedModelOptions, }; - this.reconnectEvent.set(); + + // When the WebSocket is live, send a mid-stream session.update so providers + // that support it (e.g. AssemblyAI, Deepgram Flux) apply changes without + // reconnecting. Unsupported providers ignore the message. + if (this.activeWs && this.activeWs.readyState === 1) { + const settings: Record = {}; + if (opts.model !== undefined) settings.model = opts.model; + if (opts.language !== undefined) settings.language = normalizeLanguage(opts.language); + if (opts.modelOptions !== undefined) settings.extra = opts.modelOptions; + if (Object.keys(settings).length > 0) { + try { + this.activeWs.send(JSON.stringify({ type: 'session.update', settings })); + } catch (e) { + this.#logger.debug({ err: e }, 'failed to send session.update, ws may be closing'); + } + } + } } protected async run(): Promise { @@ -762,6 +828,7 @@ export class SpeechStream extends BaseSpeechStream { try { ws = await this.stt.connectWs(this.connOptions.timeoutMs); + this.activeWs = ws; vadStream = vad?.stream() ?? null; // Use a per-connection controller so reconnect loops don't inherit a permanently-aborted signal. @@ -798,6 +865,7 @@ export class SpeechStream extends BaseSpeechStream { } finally { connController.abort(); this.abortController.signal.removeEventListener('abort', onStreamAbort); + this.activeWs = undefined; vadStream?.close(); const tasks = [sendTask, wsListenerTask, recvTask, waitReconnectTask]; if (vadTask) tasks.push(vadTask); @@ -808,6 +876,7 @@ export class SpeechStream extends BaseSpeechStream { if (this.abortController.signal.aborted) break; } finally { // Ensure cleanup even if connectWs throws + this.activeWs = undefined; resourceCleanup(); } } diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index ebfd3d35d..47a93bb04 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -34,10 +34,13 @@ import { import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js'; export type CartesiaModels = + | 'cartesia/sonic-3.5' | 'cartesia/sonic-3' | 'cartesia/sonic-2' | 'cartesia/sonic-turbo' - | 'cartesia/sonic'; + | 'cartesia/sonic' + | 'cartesia/sonic-3-latest' + | 'cartesia/sonic-latest'; export type DeepgramTTSModels = 'deepgram/aura' | 'deepgram/aura-2'; @@ -46,16 +49,20 @@ export type ElevenlabsModels = | 'elevenlabs/eleven_flash_v2_5' | 'elevenlabs/eleven_turbo_v2' | 'elevenlabs/eleven_turbo_v2_5' - | 'elevenlabs/eleven_multilingual_v2'; + | 'elevenlabs/eleven_multilingual_v2' + | 'elevenlabs/eleven_v3'; export type InworldModels = | 'inworld/inworld-tts-2' | 'inworld/inworld-tts-1.5-max' | 'inworld/inworld-tts-1.5-mini' + | 'inworld/inworld-tts-1.5' | 'inworld/inworld-tts-1-max' | 'inworld/inworld-tts-1'; -export type RimeModels = 'rime/arcana' | 'rime/coda' | 'rime/mistv2' | 'rime/mistv3'; +export type RimeModels = 'rime/arcana' | 'rime/coda' | 'rime/mistv2' | 'rime/mistv3' | 'rime/mist'; + +export type XaiTTSModels = 'xai/tts-1'; export interface CartesiaOptions { emotion?: string; @@ -126,12 +133,18 @@ export interface InworldOptions { text_normalization?: 'ON' | 'OFF'; } +export interface XaiTTSOptions { + /** Output bit rate in bits per second. */ + bit_rate?: 32000 | 64000 | 96000 | 128000 | 192000; +} + type _TTSModels = | CartesiaModels | DeepgramTTSModels | ElevenlabsModels | RimeModels - | InworldModels; + | InworldModels + | XaiTTSModels; export type TTSModels = | CartesiaModels @@ -139,6 +152,7 @@ export type TTSModels = | ElevenlabsModels | RimeModels | InworldModels + | XaiTTSModels | AnyString; export type ModelWithVoice = `${_TTSModels}:${string}` | TTSModels; @@ -153,7 +167,9 @@ export type TTSOptions = TModel extends CartesiaModels ? RimeOptions : TModel extends InworldModels ? InworldOptions - : Record; + : TModel extends XaiTTSModels + ? XaiTTSOptions + : Record; /** Parse a model string into [model, voice]. Voice is undefined if not specified. */ export function parseTTSModelString(model: string): [string, string | undefined] { @@ -591,10 +607,20 @@ export class SynthesizeStream extends BaseSynthesizeSt for await (const ev of sendTokenizerStream) { if (signal.aborted || closing) break; + // Carry per-utterance generation config so mid-stream voice/model/language + // changes ride the gateway's hot path (no reconnect needed on the active + // session). The gateway also merges `extra` into the live session. + const generationConfig: Record = {}; + if (this.opts.voice) generationConfig.voice = this.opts.voice; + if (this.opts.model) generationConfig.model = this.opts.model; + if (this.opts.language) generationConfig.language = this.opts.language; + await sendClientEvent( { type: 'input_transcript', transcript: ev.token + ' ', + generation_config: generationConfig, + extra: (this.opts.modelOptions as Record) ?? {}, }, ws, signal,