Port conversation-aware STT recognition

.changeset/stt-keyterms-context.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": minor
+---
+
+Add conversation-aware STT keyterm biasing and chat context forwarding hooks.

agents/src/stt/stt.ts

@@ -14,7 +14,7 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
-import type { TimedString } from '../voice/index.js';
+import type { ConversationItemAddedEvent, TimedString } from '../voice/index.js';
 
 /** Indicates start/middle/end of speech */
 export enum SpeechEventType {
@@ -136,6 +136,10 @@ export interface STTCapabilities {
   alignedTranscript?: 'word' | 'chunk' | false;
   /** Whether this STT supports speaker diarization. */
   diarization?: boolean;
+  /** Whether this STT supports keyterm prompting. */
+  keyterms?: boolean;
+  /** Whether this STT can natively consume conversation context. */
+  chatContext?: boolean;
 }
 
 export interface STTError {
@@ -161,6 +165,8 @@ export type STTCallbacks = {
 export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {
   abstract label: string;
   #capabilities: STTCapabilities;
+  private keytermsUnsupportedWarned = false;
+  private chatContextUnsupportedWarned = false;
 
   constructor(capabilities: STTCapabilities) {
     super();
@@ -234,6 +240,34 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
    */
   abstract stream(options?: { connOptions?: APIConnectOptions }): SpeechStream;
 
+  /** @internal */
+  _updateSessionKeyterms(keyterms: string[]): void {
+    if (!this.#capabilities.keyterms) {
+      if (!this.keytermsUnsupportedWarned) {
+        this.keytermsUnsupportedWarned = true;
+        log().warn(
+          { stt: this.label, keyterms },
+          'keyterms are not supported by this STT, ignoring keyterms update',
+        );
+      }
+      return;
+    }
+  }
+
+  /** @internal */
+  _pushConversationItem(ev: ConversationItemAddedEvent): void {
+    if (!this.#capabilities.chatContext) {
+      if (!this.chatContextUnsupportedWarned) {
+        this.chatContextUnsupportedWarned = true;
+        log().warn(
+          { stt: this.label, itemType: ev.item.type },
+          'chat context is not supported by this STT, ignoring chat context update',
+        );
+      }
+      return;
+    }
+  }
+
   async close(): Promise<void> {
     return;
   }

agents/src/voice/agent_activity.ts

@@ -96,6 +96,7 @@ import {
 import type {
   AgentState,
   AgentStateChangedEvent,
+  ConversationItemAddedEvent,
   EotPredictionEvent,
   UserTurnExceededEvent,
   _AgentBackchannelOpportunityEvent,
@@ -232,6 +233,7 @@ export class AgentActivity implements RecognitionHooks {
   private lock = new Mutex();
   private audioStream = new MultiInputStream<AudioFrame>();
   private audioStreamId?: string;
+  private sttConversationItemListener?: (ev: ConversationItemAddedEvent) => void;
 
   // default to null as None, which maps to the default provider tool choice value
   private toolChoice: ToolChoice | null = null;
@@ -548,6 +550,8 @@ export class AgentActivity implements RecognitionHooks {
       this._resolvedTurnDetection.on('metrics_collected', this.onMetricsCollected);
     }
 
+    this.agentSession._keytermDetector.on('metrics_collected', this.onMetricsCollected);
+
     // Bundled-default VAD is treated as absent when the RealtimeModel does
     // its own server-side turn detection — the realtime session is already
     // canonical and an extra audio pipeline would just pay the native model
@@ -596,6 +600,18 @@ export class AgentActivity implements RecognitionHooks {
       reuseResources.turnDetectorStream = undefined;
     }
 
+    const activeStt = this.stt;
+    if (activeStt instanceof STT) {
+      this.agentSession._keytermDetector.start(this.agentSession, activeStt);
+      if (activeStt.capabilities.chatContext) {
+        this.sttConversationItemListener = (ev) => activeStt._pushConversationItem(ev);
+        this.agentSession.on(
+          AgentSessionEventTypes.ConversationItemAdded,
+          this.sttConversationItemListener,
+        );
+      }
+    }
+
     this.started = true;
     this._resumeSchedulingTask();
 
@@ -3858,6 +3874,7 @@ export class AgentActivity implements RecognitionHooks {
   private async _pauseSchedulingTask(blockedTasks: Task<any>[]): Promise<void> {
     if (this._schedulingPaused) return;
 
+    await this.agentSession?._keytermDetector.close();
     this._schedulingPaused = true;
     this._drainBlockedTasks = blockedTasks;
     this.wakeupMainTask();
@@ -4314,6 +4331,17 @@ export class AgentActivity implements RecognitionHooks {
       this._resolvedTurnDetection.off('metrics_collected', this.onMetricsCollected);
     }
 
+    this.agentSession?._keytermDetector.off('metrics_collected', this.onMetricsCollected);
+    await this.agentSession?._keytermDetector.close();
+
+    if (this.agentSession && this.sttConversationItemListener) {
+      this.agentSession.off(
+        AgentSessionEventTypes.ConversationItemAdded,
+        this.sttConversationItemListener,
+      );
+      this.sttConversationItemListener = undefined;
+    }
+
     this.detachAudioInput();
     this.realtimeSpans?.clear();
     await this.realtimeSession?.close();

agents/src/voice/agent_session.ts

@@ -83,6 +83,11 @@ import {
   createUserStateChangedEvent,
 } from './events.js';
 import { AgentInput, AgentOutput } from './io.js';
+import {
+  KeytermDetector,
+  type KeytermsOptions,
+  resolveKeytermsOptions,
+} from './keyterm_detection.js';
 import { RecorderIO } from './recorder_io/index.js';
 import { RoomSessionTransport, SessionHost } from './remote_session.js';
 import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';
@@ -162,6 +167,7 @@ export interface InternalSessionOptions<UserData> extends AgentSessionOptions<Us
   turnHandling: InternalTurnHandlingOptions;
   useTtsAlignedTranscript: boolean;
   maxToolSteps: number;
+  keytermsOptions: KeytermsOptions;
   userAwayTimeout: number | null;
   ttsReadIdleTimeout: number;
   forwardAudioIdleTimeout: number;
@@ -175,6 +181,7 @@ export const defaultAgentSessionOptions = {
   ttsReadIdleTimeout: 10_000,
   forwardAudioIdleTimeout: 10_000,
   turnHandling: {},
+  keytermsOptions: {},
   useTtsAlignedTranscript: true,
   ttsTextTransforms: ['filter_markdown', 'filter_emoji'],
 } as const satisfies AgentSessionOptions;
@@ -288,6 +295,9 @@ export type AgentSessionOptions<UserData = UnknownUserData> = {
 
   useTtsAlignedTranscript?: boolean;
 
+  /** Keyterm biasing for STTs that accept a term list. */
+  keytermsOptions?: KeytermsOptions;
+
   /**
    * Transforms to apply to TTS input text. Built-in transforms are `filter_markdown`
    * and `filter_emoji`; pass `null` to disable text transforms.
@@ -317,6 +327,8 @@ export type AgentSessionUpdateOptions = {
    * - `TurnDetectionMode`: set the turn detection strategy to the provided value.
    */
   turnDetection?: TurnDetectionMode | null;
+  /** Replace the user-defined keyterms applied to the STT. Auto-detected keyterms are kept. */
+  keyterms?: string[];
 };
 
 type ActivityTransitionOptions = {
@@ -350,6 +362,8 @@ export class AgentSession<
   private sessionHost?: SessionHost;
 
   private _chatCtx: ChatContext;
+  /** @internal */
+  _keytermDetector: KeytermDetector;
   private _userData: UserData | undefined;
   private _userState: UserState = 'listening';
   private _agentState: AgentState = 'initializing';
@@ -512,6 +526,11 @@ export class AgentSession<
     // This is the "global" chat context, it holds the entire conversation history
     this._chatCtx = ChatContext.empty();
     this.sessionOptions = resolvedSessionOptions;
+    this.sessionOptions.keytermsOptions = resolveKeytermsOptions(opts.keytermsOptions);
+    this._keytermDetector = new KeytermDetector({
+      staticKeyterms: this.sessionOptions.keytermsOptions.keyterms,
+      options: this.sessionOptions.keytermsOptions.keytermDetection,
+    });
     this.options = legacyVoiceOptions;
     this._aecWarmupRemaining = this.sessionOptions.aecWarmupDuration ?? 0;
 
@@ -565,6 +584,10 @@ export class AgentSession<
     return { modelUsage: this._usageCollector.flatten().map(filterZeroValues) };
   }
 
+  get keyterms(): string[] {
+    return this._keytermDetector.keyterms;
+  }
+
   get useTtsAlignedTranscript(): boolean {
     return this.sessionOptions.useTtsAlignedTranscript;
   }
@@ -973,6 +996,10 @@ export class AgentSession<
       this.sessionOptions.turnHandling.turnDetection = normalizedTurnDetection;
     }
 
+    if (options.keyterms !== undefined) {
+      this._keytermDetector.setStaticKeyterms(options.keyterms);
+    }
+
     if (this.activity) {
       const activityOptions: Parameters<AgentActivity['updateOptions']>[0] = {};
       if (endpointing !== undefined) {

agents/src/voice/index.ts

@@ -31,6 +31,7 @@ export {
   type PlaybackStartedEvent,
   type TimedString,
 } from './io.js';
+export { type KeytermDetectionOptions, type KeytermsOptions } from './keyterm_detection.js';
 export * from './report.js';
 export * from './room_io/index.js';
 export { RunContext } from './run_context.js';