Skip to content

(phonic): add logic for maintaining a single ws conn and remove say() #1220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tinalenguyen merged 7 commits into from
Apr 10, 2026
Merged

(phonic): add logic for maintaining a single ws conn and remove say() #1220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8712e2c
phonic logic
tinalenguyen Apr 9, 2026
86a23e2
changeset
tinalenguyen Apr 9, 2026
ed9beb2
fix and note
tinalenguyen Apr 9, 2026
c09cce3
(phonic) Faster handoffs (#1221)
qionghuang6 Apr 9, 2026
bca9c2f
delete test example
tinalenguyen Apr 9, 2026
3711eeb
update deps versioning
toubatbrian Apr 10, 2026
ded311a
Update realtime_model.ts
toubatbrian Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/happy-yaks-bet.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@livekit/agents-plugin-phonic": patch
---

Update phonic plugin to reuse session for handoffs
5 changes: 5 additions & 0 deletions .changeset/plenty-baths-hug.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@livekit/agents': patch
---

remove rt session say logic and add phonic logic for resetting ws conn
7 changes: 0 additions & 7 deletions agents/src/llm/realtime.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,13 +161,6 @@ export abstract class RealtimeSession extends EventEmitter {
return;
}

say(
_text: string | ReadableStream<string>,
_options?: { allowInterruptions?: boolean },
): Promise<GenerationCreatedEvent> {
throw new Error(`${this.constructor.name} does not implement say(). use a TTS model instead`);
}

private async _mainTaskImpl(signal: AbortSignal): Promise<void> {
const reader = this.inputAudioStream.stream.getReader();
while (true) {
Expand Down
135 changes: 39 additions & 96 deletions agents/src/voice/agent_activity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ export class AgentActivity implements RecognitionHooks {

if (this.llm instanceof RealtimeModel) {
const rtReused = reuseResources?.rtSession !== undefined;

if (rtReused) {
this.logger.debug('reusing realtime session from previous activity');
this.realtimeSession = reuseResources!.rtSession;
Expand Down Expand Up @@ -408,27 +409,41 @@ export class AgentActivity implements RecognitionHooks {
// skip the update if the session is reused and no mid-session update is supported
// this means the content is the same as the previous session
const capabilities = this.llm.capabilities;
if (!rtReused || capabilities.midSessionInstructionsUpdate) {
if (rtReused && this.realtimeSession?.realtimeModel.provider == 'phonic') {
Comment thread
toubatbrian marked this conversation as resolved.
// if the session is being reused, then call phonic's _updateSession to send a full mid-session config update.
// otherwise, call the separate update_* functions to build the initial config.
try {
await this.realtimeSession!.updateInstructions(this.agent.instructions);
await (this.realtimeSession as any)._updateSession(
this.agent.instructions,
this.agent.chatCtx,
this.tools,
);
Comment thread
toubatbrian marked this conversation as resolved.
} catch (error) {
this.logger.error(error, 'failed to update the instructions');
this.logger.error(error, 'failed to update phonic session');
}
} else {
if (!rtReused || capabilities.midSessionInstructionsUpdate) {
try {
await this.realtimeSession!.updateInstructions(this.agent.instructions);
} catch (error) {
this.logger.error(error, 'failed to update the instructions');
}
}
}

if (!rtReused || capabilities.midSessionChatCtxUpdate) {
try {
await this.realtimeSession!.updateChatCtx(this.agent.chatCtx);
} catch (error) {
this.logger.error(error, 'failed to update the chat context');
if (!rtReused || capabilities.midSessionChatCtxUpdate) {
try {
await this.realtimeSession!.updateChatCtx(this.agent.chatCtx);
} catch (error) {
this.logger.error(error, 'failed to update the chat context');
}
}
}

if (!rtReused || capabilities.midSessionToolsUpdate) {
try {
await this.realtimeSession!.updateTools(this.tools);
} catch (error) {
this.logger.error(error, 'failed to update the tools');
if (!rtReused || capabilities.midSessionToolsUpdate) {
try {
await this.realtimeSession!.updateTools(this.tools);
} catch (error) {
this.logger.error(error, 'failed to update the tools');
}
}
}

Expand Down Expand Up @@ -819,33 +834,15 @@ export class AgentActivity implements RecognitionHooks {
allowInterruptions: defaultAllowInterruptions,
addToChatCtx = true,
} = options ?? {};
let allowInterruptions = defaultAllowInterruptions;

if (
this.llm instanceof RealtimeModel &&
this.llm.capabilities.turnDetection &&
this.tts &&
allowInterruptions === false
) {
this.logger.warn(
'the RealtimeModel uses a server-side turn detection, allowInterruptions cannot be false when using VoiceAgent.say(), ' +
'disable turnDetection in the RealtimeModel and use VAD on the AgentTask/VoiceAgent instead',
);
allowInterruptions = true;
}
const allowInterruptions = defaultAllowInterruptions;

if (
!audio &&
!this.tts &&
this.realtimeSession === undefined &&
this.agentSession.output.audio &&
this.agentSession.output.audioEnabled
) {
const modelInfo =
this.llm instanceof RealtimeModel
? 'a RealtimeSession that implements say()'
: 'a TTS model';
throw new Error(`trying to generate speech from text without ${modelInfo}`);
throw new Error('trying to generate speech from text without a TTS model');
}

const handle = SpeechHandle.create({
Expand All @@ -861,28 +858,14 @@ export class AgentActivity implements RecognitionHooks {
}),
);

let task: Task<void>;
if (this.realtimeSession !== undefined && !audio && !this.tts) {
task = this.createSpeechTask({
taskFn: (abortController: AbortController) =>
this.realtimeSayTask(handle, text, addToChatCtx, {}, abortController),
ownedSpeechHandle: handle,
name: 'AgentActivity.realtime_say',
});
} else {
task = this.createSpeechTask({
taskFn: (abortController: AbortController) =>
this.ttsTask(handle, text, addToChatCtx, {}, abortController, audio),
ownedSpeechHandle: handle,
name: 'AgentActivity.tts_say',
});
}
const task = this.createSpeechTask({
taskFn: (abortController: AbortController) =>
this.ttsTask(handle, text, addToChatCtx, {}, abortController, audio),
ownedSpeechHandle: handle,
name: 'AgentActivity.tts_say',
});

// Avoid duplicate state transitions for realtime say path: realtimeGenerationTask already
// performs end-of-speech transitions internally.
if (this.realtimeSession === undefined || audio !== undefined || this.tts) {
task.result.finally(() => this.onPipelineReplyDone());
}
task.result.finally(() => this.onPipelineReplyDone());
this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
return handle;
}
Expand Down Expand Up @@ -2901,46 +2884,6 @@ export class AgentActivity implements RecognitionHooks {
};
}

private async realtimeSayTask(
speechHandle: SpeechHandle,
text: string | ReadableStream<string>,
addToChatCtx: boolean,
modelSettings: ModelSettings,
replyAbortController: AbortController,
): Promise<void> {
speechHandleStorage.enterWith(speechHandle);

if (!this.realtimeSession) {
throw new Error('realtimeSession is not available');
}

await speechHandle.waitIfNotInterrupted([speechHandle._waitForAuthorization()]);

if (speechHandle.interrupted) {
return;
}

let generationEv: GenerationCreatedEvent;
try {
generationEv = await this.realtimeSession.say(text, {
allowInterruptions: speechHandle.allowInterruptions,
});
} catch (e) {
this.logger.error('failed to say text: %s', String(e));
// Keep state transition logic centralized so queued/planned speeches are respected.
this.onPipelineReplyDone();
return;
}

await this.realtimeGenerationTask(
speechHandle,
generationEv,
modelSettings,
replyAbortController,
addToChatCtx,
);
}

private async realtimeReplyTask({
speechHandle,
modelSettings: { toolChoice },
Expand Down
4 changes: 2 additions & 2 deletions plugins/hedra/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
},
"devDependencies": {
"@livekit/agents": "workspace:*",
"@livekit/rtc-node": "^0.13.22",
"@livekit/rtc-node": "catalog:",
"@microsoft/api-extractor": "^7.35.0",
"pino": "^8.19.0",
"tsup": "^8.3.5",
Expand All @@ -46,6 +46,6 @@
},
"peerDependencies": {
"@livekit/agents": "workspace:*",
"@livekit/rtc-node": "^0.13.22"
"@livekit/rtc-node": "catalog:"
}
}
2 changes: 1 addition & 1 deletion plugins/phonic/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"typescript": "^5.0.0"
},
"dependencies": {
"phonic": "^0.31.8"
"phonic": "^0.31.10"
},
"peerDependencies": {
"@livekit/agents": "workspace:*",
Expand Down
Loading