diff --git a/.changeset/remove-native-transcript-sync.md b/.changeset/remove-native-transcript-sync.md new file mode 100644 index 000000000..a28e1e4f9 --- /dev/null +++ b/.changeset/remove-native-transcript-sync.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents-plugin-phonic': patch +'@livekit/agents': patch +--- + +Deprecate the `nativeTranscriptSync` realtime model capability while preserving its existing transcript synchronization behavior for third-party models. Remove Phonic's redundant explicit opt-out now that it uses `stream_ahead_of_real_time` mode. diff --git a/agents/src/llm/realtime.ts b/agents/src/llm/realtime.ts index 1aef68d2b..10af53493 100644 --- a/agents/src/llm/realtime.ts +++ b/agents/src/llm/realtime.ts @@ -64,7 +64,10 @@ export interface RealtimeCapabilities { midSessionToolsUpdate?: boolean; /** Whether the tool and tool choice can be specified per response. */ perResponseToolChoice?: boolean; - /** Whether the model can synchronize generated transcript timing natively. */ + /** + * Whether the model synchronizes generated transcript timing natively. + * @deprecated Native transcript synchronization is no longer used by built-in models. + */ nativeTranscriptSync?: boolean; } diff --git a/agents/src/llm/realtime.type.test.ts b/agents/src/llm/realtime.type.test.ts new file mode 100644 index 000000000..6b4b041ff --- /dev/null +++ b/agents/src/llm/realtime.type.test.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expectTypeOf, it } from 'vitest'; +import type { RealtimeCapabilities } from './realtime.js'; + +describe('RealtimeCapabilities', () => { + it('accepts the released native transcript synchronization capability', () => { + const capabilities: RealtimeCapabilities = { + messageTruncation: true, + turnDetection: true, + userTranscription: true, + autoToolReplyGeneration: true, + audioOutput: true, + manualFunctionCalls: true, + nativeTranscriptSync: true, + }; + + expectTypeOf(capabilities.nativeTranscriptSync).toEqualTypeOf(); + }); +}); diff --git a/agents/src/voice/room_io/room_io.test.ts b/agents/src/voice/room_io/room_io.test.ts index 594879972..d526392e5 100644 --- a/agents/src/voice/room_io/room_io.test.ts +++ b/agents/src/voice/room_io/room_io.test.ts @@ -4,6 +4,7 @@ import { EventEmitter } from 'node:events'; import { afterEach, describe, expect, it, vi } from 'vitest'; import * as jobModule from '../../job.js'; +import { RealtimeModel } from '../../llm/index.js'; import { IdentityTransform } from '../../stream/identity_transform.js'; import { DEFAULT_API_CONNECT_OPTIONS } from '../../types.js'; import { AgentSessionEventTypes, CloseReason, createCloseEvent } from '../events.js'; @@ -65,14 +66,25 @@ function createFakeRoom() { }; } -function createFakeSession() { +type FakeSession = { + input: { audio: null }; + output: { audio: null; transcription: null }; + currentAgent?: { llm?: RealtimeModel }; + llm?: RealtimeModel; + on: ReturnType; + off: ReturnType; + emit: (event: string | symbol, value: unknown) => boolean; + _closeSoon: ReturnType; +}; + +function createFakeSession(llm?: RealtimeModel): FakeSession { const emitter = new EventEmitter(); return { input: { audio: null }, output: { audio: null, transcription: null }, currentAgent: undefined, - llm: undefined, + llm, on: vi.fn((event: string | symbol, listener: (...args: unknown[]) => void) => { emitter.on(event, listener); return emitter; @@ -86,6 +98,100 @@ function createFakeSession() { }; } +class FakeRealtimeModel extends RealtimeModel { + constructor(nativeTranscriptSync?: boolean) { + super({ + messageTruncation: true, + turnDetection: true, + userTranscription: true, + autoToolReplyGeneration: true, + audioOutput: true, + manualFunctionCalls: true, + nativeTranscriptSync, + }); + } + + get model(): string { + return 'fake-realtime'; + } + + session(): never { + throw new Error('not used'); + } + + async close(): Promise {} +} + +describe('RoomIO native transcript synchronization', () => { + it('disables SDK synchronization when the initial realtime model synchronizes natively', async () => { + const room = createFakeRoom(); + const session = createFakeSession(new FakeRealtimeModel(true)); + const roomIO = new RoomIO({ + // @ts-expect-error This focused test uses the minimal AgentSession surface RoomIO consumes. + agentSession: session, + // @ts-expect-error This focused test uses the minimal Room surface RoomIO consumes. + room, + inputOptions: { audioEnabled: false, textEnabled: false }, + }); + + roomIO.start(); + + const synchronizer = Reflect.get(roomIO, 'transcriptionSynchronizer'); + expect(synchronizer.enabled).toBe(false); + await roomIO.close(); + }); + + it.each([false, undefined])( + 'keeps SDK synchronization enabled when native synchronization is %s', + async (nativeTranscriptSync) => { + const room = createFakeRoom(); + const session = createFakeSession(new FakeRealtimeModel(nativeTranscriptSync)); + const roomIO = new RoomIO({ + // @ts-expect-error This focused test uses the minimal AgentSession surface RoomIO consumes. + agentSession: session, + // @ts-expect-error This focused test uses the minimal Room surface RoomIO consumes. + room, + inputOptions: { audioEnabled: false, textEnabled: false }, + }); + + roomIO.start(); + + const synchronizer = Reflect.get(roomIO, 'transcriptionSynchronizer'); + expect(synchronizer.enabled).toBe(true); + await roomIO.close(); + }, + ); + + it.each([ + { initial: true, handoff: false, expected: true }, + { initial: false, handoff: true, expected: false }, + { initial: true, handoff: undefined, expected: true }, + ])( + 'updates SDK synchronization after handoff from $initial to $handoff', + async ({ initial, handoff, expected }) => { + const room = createFakeRoom(); + const session = createFakeSession(new FakeRealtimeModel(initial)); + const roomIO = new RoomIO({ + // @ts-expect-error This focused test uses the minimal AgentSession surface RoomIO consumes. + agentSession: session, + // @ts-expect-error This focused test uses the minimal Room surface RoomIO consumes. + room, + inputOptions: { audioEnabled: false, textEnabled: false }, + }); + roomIO.start(); + + session.currentAgent = { llm: new FakeRealtimeModel(handoff) }; + session.emit(AgentSessionEventTypes.ConversationItemAdded, { + item: { type: 'agent_handoff' }, + }); + + const synchronizer = Reflect.get(roomIO, 'transcriptionSynchronizer'); + expect(synchronizer.enabled).toBe(expected); + await roomIO.close(); + }, + ); +}); + describe('RoomIO deleteRoomOnClose', () => { afterEach(() => { vi.restoreAllMocks(); diff --git a/agents/src/voice/transcription/synchronizer.test.ts b/agents/src/voice/transcription/synchronizer.test.ts index d3b100d3f..a0ea88621 100644 --- a/agents/src/voice/transcription/synchronizer.test.ts +++ b/agents/src/voice/transcription/synchronizer.test.ts @@ -5,7 +5,11 @@ import { AudioFrame } from '@livekit/rtc-node'; import { afterEach, describe, expect, it, vi } from 'vitest'; import * as logModule from '../../log.js'; import { AudioOutput, TextOutput } from '../io.js'; -import { SpeakingRateData, TranscriptionSynchronizer } from './synchronizer.js'; +import { + SpeakingRateData, + TranscriptionSynchronizer, + defaultTextSyncOptions, +} from './synchronizer.js'; describe('SpeakingRateData', () => { describe('constructor', () => { @@ -230,6 +234,21 @@ class MockTextOutput extends TextOutput { flush(): void {} } +describe('TranscriptionSynchronizer enabled behavior', () => { + it('directly forwards text when synchronization is disabled', async () => { + const downstream = new MockTextOutput(); + const synchronizer = new TranscriptionSynchronizer(new MockAudioOutput(), downstream, { + ...defaultTextSyncOptions, + enabled: false, + }); + + await synchronizer.textOutput.captureText('hello'); + + expect(downstream.captured).toEqual(['hello']); + await synchronizer.close(); + }); +}); + describe('TranscriptionSynchronizer attachment warnings', () => { const textDetachedWarning = 'TranscriptSynchronizer text output was detached while audio output is still active; ' + diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index dc5c081f2..005b88727 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -587,8 +587,8 @@ export class TranscriptionSynchronizer { _audioAttached: boolean = true; /** @internal */ _textAttached: boolean = true; - // warn once per enabled cycle when only one of audio/text is detached; reset when - // the synchronizer transitions back to enabled + // warn once per detach cycle when only one of audio/text is detached; reset when + // both outputs are reattached /** @internal */ _warnedAsymmetricDetach: boolean = false; diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index b2c3d245b..06663f4f8 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -195,7 +195,6 @@ export class RealtimeModel extends llm.RealtimeModel { midSessionInstructionsUpdate: true, midSessionToolsUpdate: true, perResponseToolChoice: false, - nativeTranscriptSync: false, }); const apiKey = options.apiKey || process.env.PHONIC_API_KEY;