Skip to content

refactor(voice): extract shared forwardedTextFor helper #1918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ShayneP merged 1 commit into from
Jun 30, 2026
Merged

refactor(voice): extract shared forwardedTextFor helper #1918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 13 additions & 25 deletions agents/src/voice/agent_activity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ import {
type _AudioOut,
type _TextOut,
applyInstructionsModality,
forwardedTextFor,
performAudioForwarding,
performLLMInference,
performTTSInference,
Expand Down Expand Up @@ -202,6 +203,15 @@ interface PausedSpeechInfo {
timeout: number;
}

/// Analog to Python's _ForwardOutput
export interface ForwardOutput {
played: 'full' | 'partial' | 'skipped';
textOut: _TextOut | null;
synchronizedTranscript?: string;
audioOut: _AudioOut | null;
playbackPositionInS: number;
}

export class AgentActivity implements RecognitionHooks {
agent: Agent;
agentSession: AgentSession;
Expand Down Expand Up @@ -2582,21 +2592,7 @@ export class AgentActivity implements RecognitionHooks {
ttsGenData?: _TTSGenerationData;
}

interface SegmentOutput {
textOut: _TextOut | null;
audioOut: _AudioOut | null;
played: 'full' | 'partial' | 'skipped';
playbackPositionInS: number;
synchronizedTranscript?: string;
}

const forwardedTextFor = (output: SegmentOutput): string => {
if (output.played === 'skipped') return '';
if (output.played === 'partial' && output.audioOut) {
return output.synchronizedTranscript || output.textOut?.text || '';
}
return output.textOut?.text ?? '';
};
type SegmentOutput = ForwardOutput;

const segmentQueue = new AsyncIterableQueue<SpeechSegment>();
let synthesizeTask: Task<void> | null = null;
Expand Down Expand Up @@ -3239,14 +3235,9 @@ export class AgentActivity implements RecognitionHooks {
}
};

interface MessageOutput {
interface MessageOutput extends ForwardOutput {
message: MessageGeneration;
textOut: _TextOut | null;
audioOut: _AudioOut | null;
modalities?: ('text' | 'audio')[];
played: 'full' | 'partial' | 'skipped';
playbackPositionInS: number;
synchronizedTranscript?: string;
}

const tasks: Array<Task<void>> = [];
Expand Down Expand Up @@ -3425,10 +3416,7 @@ export class AgentActivity implements RecognitionHooks {
if (output.played === 'skipped') continue;

const interrupted = output.played === 'partial';
let forwardedText = output.textOut?.text || '';
if (interrupted && output.audioOut) {
forwardedText = output.synchronizedTranscript || forwardedText;
}
const forwardedText = forwardedTextFor(output);

if (interrupted && realtimeModel.capabilities.messageTruncation) {
// Defense-in-depth: playbackPositionInS can be non-finite when the avatar
Expand Down
17 changes: 17 additions & 0 deletions agents/src/voice/generation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
functionCallStorage,
isStopResponse,
} from './agent.js';
import type { ForwardOutput } from './agent_activity.ts';
import type { AgentSession } from './agent_session.js';
import {
AudioOutput,
Expand Down Expand Up @@ -831,6 +832,22 @@ export interface _AudioOut {
startedForwardingAt?: number;
}

/**
* The text that actually reached the user, accounting for interruptions.
*
* On a mid-playout interruption (`played === 'partial'`) we prefer the
* playback-aligned `synchronizedTranscript`, but fall back to the full generated
* text when no synchronized transcript is available (e.g. avatar outputs) so the
* heard reply is still committed to chat ctx rather than dropped.
*/
export function forwardedTextFor(output: ForwardOutput): string {
if (output.played === 'skipped') return '';
if (output.played === 'partial' && output.synchronizedTranscript) {
return output.synchronizedTranscript;
}
return output.textOut?.text ?? '';
}

async function forwardAudio(
ttsStream: ReadableStream<AudioFrame>,
audioOutput: AudioOutput,
Expand Down