fix(lemonslice): wait for playback start

.changeset/witty-lemons-dance.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-lemonslice": patch
+---
+
+Wait for LemonSlice avatar playback-start notifications before marking agent audio as playing.

examples/src/lemonslice_realtime_avatar.ts

@@ -42,8 +42,13 @@ export default defineAgent({
         }),
         turnDetection: new livekit.turnDetector.MultilingualModel(),
         vad: ctx.proc.userData.vad! as silero.VAD,
-        voiceOptions: {
-          preemptiveGeneration: true,
+        turnHandling: {
+          interruption: {
+            resumeFalseInterruption: false,
+          },
+          preemptiveGeneration: {
+            enabled: true,
+          },
         },
       });
 

plugins/lemonslice/src/avatar.test.ts

@@ -2,9 +2,14 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { initializeLogger, voice } from '@livekit/agents';
+import { type Room, TrackKind } from '@livekit/rtc-node';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { AvatarSession } from './avatar.js';
 
+type DataStreamAudioOutputInternals = {
+  waitPlaybackStart: boolean;
+};
+
 describe('LemonSlice AvatarSession', () => {
   beforeEach(() => {
     initializeLogger({ pretty: false });
@@ -13,6 +18,10 @@ describe('LemonSlice AvatarSession', () => {
 
   afterEach(() => {
     vi.restoreAllMocks();
+    voice.DataStreamAudioOutput._playbackFinishedRpcRegistered = false;
+    voice.DataStreamAudioOutput._playbackFinishedHandlers = {};
+    voice.DataStreamAudioOutput._playbackStartedRpcRegistered = false;
+    voice.DataStreamAudioOutput._playbackStartedHandlers = {};
   });
 
   it('merges extraPayload into the session creation request body', async () => {
@@ -98,8 +107,57 @@ describe('LemonSlice AvatarSession', () => {
     });
 
     await expect(
-      avatar.start({ _started: false, output: { audio: null } } as any, {} as any),
+      avatar.start(
+        { _started: false, output: { audio: null } } as unknown as voice.AgentSession,
+        {} as unknown as Room,
+      ),
     ).rejects.toThrow('super-start-called');
     expect(superStartSpy).toHaveBeenCalledTimes(1);
   });
+
+  it('configures DataStreamAudioOutput to wait for remote playback started', async () => {
+    vi.spyOn(voice.AvatarSession.prototype, 'start').mockResolvedValue(undefined);
+
+    const avatar = new AvatarSession({
+      apiKey: 'test-api-key',
+      agentImageUrl: 'https://example.com/avatar.png',
+    });
+    vi.spyOn(
+      avatar as unknown as {
+        startAgent(livekitUrl: string, livekitToken: string): Promise<string>;
+      },
+      'startAgent',
+    ).mockResolvedValue('test-session-id');
+
+    const remoteParticipant = {
+      identity: 'lemonslice-avatar-agent',
+      trackPublications: new Map([['video', { kind: TrackKind.KIND_VIDEO }]]),
+    };
+    const room = {
+      name: 'test-room',
+      isConnected: true,
+      localParticipant: {
+        identity: 'local-agent',
+        registerRpcMethod: vi.fn(),
+      },
+      remoteParticipants: new Map([[remoteParticipant.identity, remoteParticipant]]),
+      on: vi.fn(),
+      off: vi.fn(),
+    };
+    const agentSession = {
+      _started: false,
+      output: { audio: null },
+    } as unknown as voice.AgentSession;
+
+    const sessionId = await avatar.start(agentSession, room as unknown as Room, {
+      livekitUrl: 'wss://livekit.example.com',
+      livekitApiKey: 'livekit-api-key',
+      livekitApiSecret: 'livekit-api-secret',
+    });
+
+    expect(sessionId).toBe('test-session-id');
+    const audioOutput = agentSession.output.audio;
+    expect(audioOutput).toBeInstanceOf(voice.DataStreamAudioOutput);
+    expect((audioOutput as unknown as DataStreamAudioOutputInternals).waitPlaybackStart).toBe(true);
+  });
 });

plugins/lemonslice/src/avatar.ts

@@ -249,6 +249,7 @@ export class AvatarSession extends voice.AvatarSession {
       destinationIdentity: this.avatarParticipantIdentity,
       sampleRate: SAMPLE_RATE,
       waitRemoteTrack: TrackKind.KIND_VIDEO,
+      waitPlaybackStart: true,
     });
 
     return sessionId;