diff --git a/.changeset/witty-lemons-dance.md b/.changeset/witty-lemons-dance.md new file mode 100644 index 000000000..bbb19da4c --- /dev/null +++ b/.changeset/witty-lemons-dance.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents-plugin-lemonslice": patch +--- + +Wait for LemonSlice avatar playback-start notifications before marking agent audio as playing. diff --git a/examples/src/lemonslice_realtime_avatar.ts b/examples/src/lemonslice_realtime_avatar.ts index 94c0d888d..b2afc544b 100644 --- a/examples/src/lemonslice_realtime_avatar.ts +++ b/examples/src/lemonslice_realtime_avatar.ts @@ -42,8 +42,13 @@ export default defineAgent({ }), turnDetection: new livekit.turnDetector.MultilingualModel(), vad: ctx.proc.userData.vad! as silero.VAD, - voiceOptions: { - preemptiveGeneration: true, + turnHandling: { + interruption: { + resumeFalseInterruption: false, + }, + preemptiveGeneration: { + enabled: true, + }, }, }); diff --git a/plugins/lemonslice/src/avatar.test.ts b/plugins/lemonslice/src/avatar.test.ts index 230f8a288..d31957dca 100644 --- a/plugins/lemonslice/src/avatar.test.ts +++ b/plugins/lemonslice/src/avatar.test.ts @@ -2,9 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 import { initializeLogger, voice } from '@livekit/agents'; +import { type Room, TrackKind } from '@livekit/rtc-node'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { AvatarSession } from './avatar.js'; +type DataStreamAudioOutputInternals = { + waitPlaybackStart: boolean; +}; + describe('LemonSlice AvatarSession', () => { beforeEach(() => { initializeLogger({ pretty: false }); @@ -13,6 +18,10 @@ describe('LemonSlice AvatarSession', () => { afterEach(() => { vi.restoreAllMocks(); + voice.DataStreamAudioOutput._playbackFinishedRpcRegistered = false; + voice.DataStreamAudioOutput._playbackFinishedHandlers = {}; + voice.DataStreamAudioOutput._playbackStartedRpcRegistered = false; + voice.DataStreamAudioOutput._playbackStartedHandlers = {}; }); it('merges extraPayload into the session creation request body', async () => { @@ -98,8 +107,57 @@ describe('LemonSlice AvatarSession', () => { }); await expect( - avatar.start({ _started: false, output: { audio: null } } as any, {} as any), + avatar.start( + { _started: false, output: { audio: null } } as unknown as voice.AgentSession, + {} as unknown as Room, + ), ).rejects.toThrow('super-start-called'); expect(superStartSpy).toHaveBeenCalledTimes(1); }); + + it('configures DataStreamAudioOutput to wait for remote playback started', async () => { + vi.spyOn(voice.AvatarSession.prototype, 'start').mockResolvedValue(undefined); + + const avatar = new AvatarSession({ + apiKey: 'test-api-key', + agentImageUrl: 'https://example.com/avatar.png', + }); + vi.spyOn( + avatar as unknown as { + startAgent(livekitUrl: string, livekitToken: string): Promise; + }, + 'startAgent', + ).mockResolvedValue('test-session-id'); + + const remoteParticipant = { + identity: 'lemonslice-avatar-agent', + trackPublications: new Map([['video', { kind: TrackKind.KIND_VIDEO }]]), + }; + const room = { + name: 'test-room', + isConnected: true, + localParticipant: { + identity: 'local-agent', + registerRpcMethod: vi.fn(), + }, + remoteParticipants: new Map([[remoteParticipant.identity, remoteParticipant]]), + on: vi.fn(), + off: vi.fn(), + }; + const agentSession = { + _started: false, + output: { audio: null }, + } as unknown as voice.AgentSession; + + const sessionId = await avatar.start(agentSession, room as unknown as Room, { + livekitUrl: 'wss://livekit.example.com', + livekitApiKey: 'livekit-api-key', + livekitApiSecret: 'livekit-api-secret', + }); + + expect(sessionId).toBe('test-session-id'); + const audioOutput = agentSession.output.audio; + expect(audioOutput).toBeInstanceOf(voice.DataStreamAudioOutput); + expect((audioOutput as unknown as DataStreamAudioOutputInternals).waitPlaybackStart).toBe(true); + }); }); diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index accbef84c..333b89083 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -249,6 +249,7 @@ export class AvatarSession extends voice.AvatarSession { destinationIdentity: this.avatarParticipantIdentity, sampleRate: SAMPLE_RATE, waitRemoteTrack: TrackKind.KIND_VIDEO, + waitPlaybackStart: true, }); return sessionId;