diff --git a/.changeset/scope-forward-audio-playback-started.md b/.changeset/scope-forward-audio-playback-started.md new file mode 100644 index 000000000..5c5cd277e --- /dev/null +++ b/.changeset/scope-forward-audio-playback-started.md @@ -0,0 +1,15 @@ +--- +'@livekit/agents': patch +--- + +fix(voice): scope forwardAudio's playback-started listener to its own segment + +When a speech is interrupted, the scheduling loop immediately authorizes the next +speech, so the new segment's `forwardAudio` registers its `playback_started` +listener on the shared audio output while the interrupted segment is still +emitting events during teardown. The stray event resolved the new segment's +`firstFrameFut` before its first frame was captured, which skipped resampler +creation and pushed an unresampled frame straight to the `AudioSource` +(`RtcError: sample_rate and num_channels don't match`) and corrupted playback +bookkeeping. The listener now only resolves `firstFrameFut` after the segment has +captured its own first frame. diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 1ea3bf697..05e3555af 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -836,8 +836,14 @@ async function forwardAudio( const reader = ttsStream.getReader(); let resampler: AudioResampler | null = null; + // The audio output is shared across overlapping segments, so ignore a + // PLAYBACK_STARTED from another segment until we capture our own first frame. + // Resolving `firstFrameFut` early skips resampler creation and pushes an + // unresampled frame (`RtcError: sample_rate and num_channels don't match`). + let hasCapturedOwnFrame = false; + const onPlaybackStarted = (ev: { createdAt: number }) => { - if (!out.firstFrameFut.done) { + if (hasCapturedOwnFrame && !out.firstFrameFut.done) { out.firstFrameFut.resolve(ev.createdAt); } }; @@ -868,6 +874,10 @@ async function forwardAudio( resampler = new AudioResampler(frame.sampleRate, audioOutput.sampleRate, 1); } + // Mark before capturing so the PLAYBACK_STARTED emitted synchronously inside + // the first captureFrame is attributed to this segment. + hasCapturedOwnFrame = true; + if (resampler) { for (const f of resampler.push(frame)) { await audioOutput.captureFrame(f); diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index c7d2c0717..cbe636ebe 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -105,6 +105,60 @@ describe('TTS stream idle timeout', () => { expect(audioOut.firstFrameFut.done).toBe(true); }); + it('ignores PLAYBACK_STARTED from another segment before its own first frame', async () => { + // Stalled stream so the forwarder is still waiting on its first read when a + // stray event (from an interrupted overlapping segment) arrives; the idle + // timeout then ends the loop without this segment ever capturing a frame. + const stalledStream = new ReadableStream({ start() {} }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stalledStream, audioOutput, controller, 500); + + // Reject path is expected (no first frame ever captured). + audioOut.firstFrameFut.await.catch(() => {}); + + vi.useFakeTimers(); + + audioOutput.onPlaybackStarted(Date.now()); + expect(audioOut.firstFrameFut.done).toBe(false); + + const taskPromise = task.result; + await vi.advanceTimersByTimeAsync(600); + await taskPromise; + + vi.useRealTimers(); + + expect(audioOutput.capturedFrames.length).toBe(0); + expect(audioOut.firstFrameFut.rejected).toBe(true); + }); + + it('resamples a rate-mismatched frame even after a stray PLAYBACK_STARTED', async () => { + // Output is 24kHz; frames are 16kHz and must be resampled regardless of any + // stray PLAYBACK_STARTED resolving firstFrameFut early. + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(createSilentFrame(16000)); + controller.enqueue(createSilentFrame(16000)); + controller.close(); + }, + }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + + audioOutput.onPlaybackStarted(Date.now()); + + await task.result; + + expect(audioOut.firstFrameFut.done).toBe(true); + expect(audioOutput.capturedFrames.length).toBeGreaterThan(0); + for (const f of audioOutput.capturedFrames) { + expect(f.sampleRate).toBe(24000); + } + }); + it('performTTSInference completes when TTS node returns stalled stream', async () => { const stalledTtsStream = new ReadableStream({ start(controller) {