Skip to content

feat(inference): enhance schemas and models for TTS and STT #1680

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
toubatbrian merged 1 commit into from
Jun 1, 2026
Merged

feat(inference): enhance schemas and models for TTS and STT #1680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions agents/src/inference/api_protos.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,17 @@ export const ttsSessionCreateEventSchema = z.object({
transcript: z.string().optional(),
});

export const ttsGenerationConfigSchema = z.object({
voice: z.string().optional(),
language: z.string().optional(),
model: z.string().optional(),
});

export const ttsInputTranscriptEventSchema = z.object({
type: z.literal('input_transcript'),
transcript: z.string(),
generation_config: ttsGenerationConfigSchema.optional(),
extra: z.record(z.string(), z.unknown()).optional(),
});

export const ttsSessionFlushEventSchema = z.object({
Expand Down Expand Up @@ -109,6 +117,7 @@ export const ttsServerEventSchema = z.union([
ttsUnknownServerEventSchema,
]);

export type TtsGenerationConfig = z.infer<typeof ttsGenerationConfigSchema>;
export type TtsSessionCreateEvent = z.infer<typeof ttsSessionCreateEventSchema>;
export type TtsInputTranscriptEvent = z.infer<typeof ttsInputTranscriptEventSchema>;
export type TtsSessionFlushEvent = z.infer<typeof ttsSessionFlushEventSchema>;
Expand Down
7 changes: 7 additions & 0 deletions agents/src/inference/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@ export {
type InferenceLLMOptions,
type LLMModels,
type XAIModels,
type ZAIModels,
} from './llm.js';

export {
normalizeSTTFallback,
parseSTTModelString,
STT,
type DeepgramFluxModels,
type DeepgramFluxOptions,
type InworldSTTModels,
type InworldSTTOptions,
type STTFallbackModel,
type STTFallbackModelType,
type STTLanguages,
Expand All @@ -39,6 +44,8 @@ export {
type TTSModels,
type ModelWithVoice as TTSModelString,
type TTSOptions,
type XaiTTSModels,
type XaiTTSOptions,
} from './tts.js';

export { llm, stt, tts };
25 changes: 14 additions & 11 deletions agents/src/inference/llm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ import {
} from './utils.js';

export type OpenAIModels =
| 'openai/gpt-5.5'
| 'openai/gpt-5.4'
| 'openai/gpt-5.4-mini'
| 'openai/gpt-5.4-nano'
| 'openai/gpt-5.3-chat-latest'
| 'openai/gpt-5.2'
| 'openai/gpt-5.2-chat-latest'
Expand All @@ -31,21 +34,24 @@ export type OpenAIModels =
| 'openai/gpt-4.1-nano'
| 'openai/gpt-4o'
| 'openai/gpt-4o-mini'
| 'openai/chat-latest'
| 'openai/gpt-oss-120b';

export type GoogleModels =
| 'google/gemini-3-pro'
| 'google/gemini-3.1-pro'
| 'google/gemini-3-flash'
| 'google/gemini-3.1-flash-lite'
| 'google/gemini-3.5-flash'
| 'google/gemini-2.5-pro'
| 'google/gemini-2.5-flash'
| 'google/gemini-2.5-flash-lite'
| 'google/gemini-2.0-flash'
| 'google/gemini-2.0-flash-lite';
| 'google/gemini-2.5-flash-lite';

export type MoonshotModels = 'moonshotai/kimi-k2-instruct';
export type MoonshotModels = 'moonshotai/kimi-k2.5' | 'moonshotai/kimi-k2.6';

export type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2';

export type ZAIModels = 'zai/glm-5.1';

export type XAIModels =
| 'xai/grok-4-1-fast-non-reasoning'
| 'xai/grok-4-1-fast-reasoning'
Expand All @@ -57,6 +63,7 @@ type ChatCompletionPredictionContentParam =
Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;
type WebSearchOptions = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions>;
type ToolChoice = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice']>;
type ResponseFormat = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']>;
type Verbosity = 'low' | 'medium' | 'high';

export interface ChatCompletionOptions extends Record<string, unknown> {
Expand Down Expand Up @@ -88,15 +95,15 @@ export interface ChatCompletionOptions extends Record<string, unknown> {

// livekit-typed arguments
tool_choice?: ToolChoice;
// TODO(brian): support response format
// response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']
response_format?: ResponseFormat;
}

export type LLMModels =
| OpenAIModels
| GoogleModels
| MoonshotModels
| DeepSeekModels
| ZAIModels
| XAIModels
| AnyString;

Expand Down Expand Up @@ -275,7 +282,6 @@ export class LLM extends llm.LLM {
parallelToolCalls,
toolChoice,
inferenceClass,
// TODO(AJS-270): Add response_format parameter support
extraKwargs,
}: {
chatCtx: llm.ChatContext;
Expand All @@ -284,7 +290,6 @@ export class LLM extends llm.LLM {
parallelToolCalls?: boolean;
toolChoice?: llm.ToolChoice;
inferenceClass?: InferenceClass;
// TODO(AJS-270): Add responseFormat parameter
extraKwargs?: Record<string, unknown>;
}): LLMStream {
let modelOptions: Record<string, unknown> = { ...(extraKwargs || {}) };
Expand All @@ -310,8 +315,6 @@ export class LLM extends llm.LLM {
const resolvedInferenceClass =
inferenceClass !== undefined ? inferenceClass : this.opts.inferenceClass;

// TODO(AJS-270): Add response_format support here

modelOptions = { ...modelOptions, ...this.opts.modelOptions };

return new LLMStream(this, {
Expand Down
103 changes: 86 additions & 17 deletions agents/src/inference/stt.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,33 @@ import {
import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';

export type DeepgramModels =
| 'deepgram/flux-general'
| 'deepgram/nova-3'
| 'deepgram/nova-3-medical'
| 'deepgram/nova-2'
| 'deepgram/nova-2-medical'
| 'deepgram/nova-2-conversationalai'
| 'deepgram/nova-2-phonecall';

export type CartesiaModels = 'cartesia/ink-whisper';
export type DeepgramFluxModels =
| 'deepgram/flux-general'
| 'deepgram/flux-general-en'
| 'deepgram/flux-general-multi';

export type CartesiaModels = 'cartesia/ink-whisper' | 'cartesia/ink-2' | 'cartesia/ink-2-latest';

export type AssemblyaiModels =
| 'assemblyai/universal-streaming'
| 'assemblyai/universal-streaming-multilingual';
| 'assemblyai/universal-streaming-multilingual'
| 'assemblyai/u3-rt-pro';

export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime';

export type XaiSTTModels = 'xai/stt-1';

export type SpeechmaticsModels = 'speechmatics/enhanced' | 'speechmatics/standard';

export type InworldSTTModels = 'inworld/inworld-stt-1';

export interface CartesiaOptions {
/** Minimum volume threshold. Default: not specified. */
min_volume?: number;
Expand Down Expand Up @@ -79,8 +86,23 @@ export interface DeepgramOptions {
mip_opt_out?: boolean;
/** Enable speaker diarization. Default: false. */
diarize?: boolean;
/** Eager end-of-turn threshold (0.0–1.0). Enables preflight transcripts for preemptive generation. */
}

export interface DeepgramFluxOptions {
/** Eager end-of-turn threshold (0.3–0.9). Enables preflight transcripts for preemptive generation. Default: 0.5. */
eager_eot_threshold?: number;
/** End-of-turn threshold (0.5–0.9). */
eot_threshold?: number;
/** End-of-turn timeout in milliseconds. */
eot_timeout_ms?: number;
/** Key terms for recognition. */
keyterm?: string | string[];
/** Opt out of model improvement program. Default: false. */
mip_opt_out?: boolean;
/** Language hint. */
language_hint?: string;
/** Enable automatic language detection. */
detect_language?: boolean;
}

export interface AssemblyAIOptions {
Expand Down Expand Up @@ -148,6 +170,27 @@ export interface SpeechmaticsOptions {
transcript_filtering_config?: Record<string, unknown>;
}

export interface InworldSTTOptions {
/** Enable Voice Profile detection. Default: true. */
enable_voice_profile?: boolean;
/** Max labels per category in voice-profile responses (1–20). Default: 10. */
voice_profile_top_n?: number;
/** Enable word-level timestamps. Default: true. */
include_word_timestamps?: boolean;
/** Wire-format encoding sent to Inworld. Default: LINEAR16. */
audio_encoding?: 'LINEAR16' | 'AUTO_DETECT';
/** Stop transcription after this many seconds of silence; 0 disables. */
inactivity_timeout_seconds?: number;
/** End-of-turn confidence threshold (0.0–1.0). Default: 0.5. */
end_of_turn_confidence_threshold?: number;
/** Domain-specific contextual hints passed to the model. */
prompts?: string[];
/** Minimum end-of-turn silence in milliseconds when confident. */
min_end_of_turn_silence_when_confident?: number;
/** VAD threshold (0.0–1.0). Default: 0.5. */
vad_threshold?: number;
}

export type STTLanguages =
| 'multi'
| 'en'
Expand All @@ -173,27 +216,33 @@ function diarizationEnabled(extraKwargs: Record<string, unknown> | undefined): b

type _STTModels =
| DeepgramModels
| DeepgramFluxModels
| CartesiaModels
| AssemblyaiModels
| ElevenlabsSTTModels
| XaiSTTModels
| SpeechmaticsModels;
| SpeechmaticsModels
| InworldSTTModels;

export type STTModels = _STTModels | 'auto' | AnyString;

export type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels;

export type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels
? DeepgramOptions
: TModel extends CartesiaModels
? CartesiaOptions
: TModel extends AssemblyaiModels
? AssemblyAIOptions
: TModel extends XaiSTTModels
? XaiOptions
: TModel extends SpeechmaticsModels
? SpeechmaticsOptions
: Record<string, unknown>;
export type STTOptions<TModel extends STTModels> = TModel extends DeepgramFluxModels
? DeepgramFluxOptions
: TModel extends DeepgramModels
? DeepgramOptions
: TModel extends CartesiaModels
? CartesiaOptions
: TModel extends AssemblyaiModels
? AssemblyAIOptions
: TModel extends XaiSTTModels
? XaiOptions
: TModel extends SpeechmaticsModels
? SpeechmaticsOptions
: TModel extends InworldSTTModels
? InworldSTTOptions
: Record<string, unknown>;

/** Inference Fallback Adapter: configuration for a fallback STT model that runs server-side in LiveKit Inference, providing automatic fallback between providers. Extra fields are passed through to the provider. */
export interface STTFallbackModel {
Expand Down Expand Up @@ -532,6 +581,7 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
private reconnectEvent = new Event();
private stt: STT<TModel>;
private connOptions: APIConnectOptions;
private activeWs?: WebSocket;

#logger = log();

Expand Down Expand Up @@ -563,7 +613,23 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,
modelOptions: mergedModelOptions,
};
this.reconnectEvent.set();

// When the WebSocket is live, send a mid-stream session.update so providers
// that support it (e.g. AssemblyAI, Deepgram Flux) apply changes without
// reconnecting. Unsupported providers ignore the message.
if (this.activeWs && this.activeWs.readyState === 1) {
const settings: Record<string, unknown> = {};
if (opts.model !== undefined) settings.model = opts.model;
if (opts.language !== undefined) settings.language = normalizeLanguage(opts.language);
if (opts.modelOptions !== undefined) settings.extra = opts.modelOptions;
Comment thread
toubatbrian marked this conversation as resolved.
if (Object.keys(settings).length > 0) {
try {
this.activeWs.send(JSON.stringify({ type: 'session.update', settings }));
} catch (e) {
this.#logger.debug({ err: e }, 'failed to send session.update, ws may be closing');
}
}
}
}

protected async run(): Promise<void> {
Expand Down Expand Up @@ -762,6 +828,7 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {

try {
ws = await this.stt.connectWs(this.connOptions.timeoutMs);
this.activeWs = ws;
vadStream = vad?.stream() ?? null;

// Use a per-connection controller so reconnect loops don't inherit a permanently-aborted signal.
Expand Down Expand Up @@ -798,6 +865,7 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
} finally {
connController.abort();
this.abortController.signal.removeEventListener('abort', onStreamAbort);
this.activeWs = undefined;
vadStream?.close();
const tasks = [sendTask, wsListenerTask, recvTask, waitReconnectTask];
if (vadTask) tasks.push(vadTask);
Expand All @@ -808,6 +876,7 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
if (this.abortController.signal.aborted) break;
} finally {
// Ensure cleanup even if connectWs throws
this.activeWs = undefined;
resourceCleanup();
}
}
Expand Down
Loading
Loading