From 40bb87f7506c0e3abb27967eb92c7231c7ef5bab Mon Sep 17 00:00:00 2001
From: Jason Lernerman <jason.lernerman@livekit.io>
Date: Tue, 30 Jun 2026 11:53:29 -0400
Subject: [PATCH] refactor(voice): extract shared forwardedTextFor helper

Hoist the per-segment forwarded-text logic into a single forwardedTextFor
helper (mirroring Python's _ForwardOutput.forwarded_text) used by both the
pipeline and realtime reply paths, replacing the duplicated inline logic.
Introduce a shared ForwardOutput interface backing SegmentOutput and
MessageOutput.
---
 agents/src/voice/agent_activity.ts | 38 ++++++++++--------------------
 agents/src/voice/generation.ts     | 17 +++++++++++++
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts
index 13160ed91..412a7aab0 100644
--- a/agents/src/voice/agent_activity.ts
+++ b/agents/src/voice/agent_activity.ts
@@ -115,6 +115,7 @@ import {
   type _AudioOut,
   type _TextOut,
   applyInstructionsModality,
+  forwardedTextFor,
   performAudioForwarding,
   performLLMInference,
   performTTSInference,
@@ -202,6 +203,15 @@ interface PausedSpeechInfo {
   timeout: number;
 }
 
+/// Analog to Python's _ForwardOutput
+export interface ForwardOutput {
+  played: 'full' | 'partial' | 'skipped';
+  textOut: _TextOut | null;
+  synchronizedTranscript?: string;
+  audioOut: _AudioOut | null;
+  playbackPositionInS: number;
+}
+
 export class AgentActivity implements RecognitionHooks {
   agent: Agent;
   agentSession: AgentSession;
@@ -2582,21 +2592,7 @@ export class AgentActivity implements RecognitionHooks {
       ttsGenData?: _TTSGenerationData;
     }
 
-    interface SegmentOutput {
-      textOut: _TextOut | null;
-      audioOut: _AudioOut | null;
-      played: 'full' | 'partial' | 'skipped';
-      playbackPositionInS: number;
-      synchronizedTranscript?: string;
-    }
-
-    const forwardedTextFor = (output: SegmentOutput): string => {
-      if (output.played === 'skipped') return '';
-      if (output.played === 'partial' && output.audioOut) {
-        return output.synchronizedTranscript || output.textOut?.text || '';
-      }
-      return output.textOut?.text ?? '';
-    };
+    type SegmentOutput = ForwardOutput;
 
     const segmentQueue = new AsyncIterableQueue<SpeechSegment>();
     let synthesizeTask: Task<void> | null = null;
@@ -3239,14 +3235,9 @@ export class AgentActivity implements RecognitionHooks {
       }
     };
 
-    interface MessageOutput {
+    interface MessageOutput extends ForwardOutput {
       message: MessageGeneration;
-      textOut: _TextOut | null;
-      audioOut: _AudioOut | null;
       modalities?: ('text' | 'audio')[];
-      played: 'full' | 'partial' | 'skipped';
-      playbackPositionInS: number;
-      synchronizedTranscript?: string;
     }
 
     const tasks: Array<Task<void>> = [];
@@ -3425,10 +3416,7 @@ export class AgentActivity implements RecognitionHooks {
         if (output.played === 'skipped') continue;
 
         const interrupted = output.played === 'partial';
-        let forwardedText = output.textOut?.text || '';
-        if (interrupted && output.audioOut) {
-          forwardedText = output.synchronizedTranscript || forwardedText;
-        }
+        const forwardedText = forwardedTextFor(output);
 
         if (interrupted && realtimeModel.capabilities.messageTruncation) {
           // Defense-in-depth: playbackPositionInS can be non-finite when the avatar
diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts
index 6cf12d6ea..e438b25ce 100644
--- a/agents/src/voice/generation.ts
+++ b/agents/src/voice/generation.ts
@@ -47,6 +47,7 @@ import {
   functionCallStorage,
   isStopResponse,
 } from './agent.js';
+import type { ForwardOutput } from './agent_activity.ts';
 import type { AgentSession } from './agent_session.js';
 import {
   AudioOutput,
@@ -831,6 +832,22 @@ export interface _AudioOut {
   startedForwardingAt?: number;
 }
 
+/**
+ * The text that actually reached the user, accounting for interruptions.
+ *
+ * On a mid-playout interruption (`played === 'partial'`) we prefer the
+ * playback-aligned `synchronizedTranscript`, but fall back to the full generated
+ * text when no synchronized transcript is available (e.g. avatar outputs) so the
+ * heard reply is still committed to chat ctx rather than dropped.
+ */
+export function forwardedTextFor(output: ForwardOutput): string {
+  if (output.played === 'skipped') return '';
+  if (output.played === 'partial' && output.synchronizedTranscript) {
+    return output.synchronizedTranscript;
+  }
+  return output.textOut?.text ?? '';
+}
+
 async function forwardAudio(
   ttsStream: ReadableStream<AudioFrame>,
   audioOutput: AudioOutput,