From 8c018d8f598a0719a17636b1354a4dd7d4f162e1 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Thu, 11 Jun 2026 17:15:57 +0100 Subject: [PATCH 1/2] fix(voice): scope forwardAudio playback-started listener to its own segment (#1760) Co-authored-by: Cursor --- .../scope-forward-audio-playback-started.md | 15 +++++ agents/src/voice/generation.ts | 17 +++++- .../src/voice/generation_tts_timeout.test.ts | 57 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 .changeset/scope-forward-audio-playback-started.md diff --git a/.changeset/scope-forward-audio-playback-started.md b/.changeset/scope-forward-audio-playback-started.md new file mode 100644 index 000000000..5c5cd277e --- /dev/null +++ b/.changeset/scope-forward-audio-playback-started.md @@ -0,0 +1,15 @@ +--- +'@livekit/agents': patch +--- + +fix(voice): scope forwardAudio's playback-started listener to its own segment + +When a speech is interrupted, the scheduling loop immediately authorizes the next +speech, so the new segment's `forwardAudio` registers its `playback_started` +listener on the shared audio output while the interrupted segment is still +emitting events during teardown. The stray event resolved the new segment's +`firstFrameFut` before its first frame was captured, which skipped resampler +creation and pushed an unresampled frame straight to the `AudioSource` +(`RtcError: sample_rate and num_channels don't match`) and corrupted playback +bookkeeping. The listener now only resolves `firstFrameFut` after the segment has +captured its own first frame. diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 1ea3bf697..a374e219b 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -836,8 +836,19 @@ async function forwardAudio( const reader = ttsStream.getReader(); let resampler: AudioResampler | null = null; + // The audio output is shared across overlapping segments. When a speech is + // interrupted, the main loop immediately authorizes the next speech, so this + // forwarder can register its listener while the interrupted segment's teardown + // is still emitting PLAYBACK_STARTED on the same output. Only honor the event + // once this loop has captured its own first frame, so a stray event from + // another segment can't resolve our `firstFrameFut` prematurely. A premature + // resolution skips resampler creation (gated on `!firstFrameFut.done`) and + // pushes an unresampled frame to the AudioSource, raising + // `RtcError: sample_rate and num_channels don't match`. + let hasCapturedOwnFrame = false; + const onPlaybackStarted = (ev: { createdAt: number }) => { - if (!out.firstFrameFut.done) { + if (hasCapturedOwnFrame && !out.firstFrameFut.done) { out.firstFrameFut.resolve(ev.createdAt); } }; @@ -868,6 +879,10 @@ async function forwardAudio( resampler = new AudioResampler(frame.sampleRate, audioOutput.sampleRate, 1); } + // Mark before capturing so the PLAYBACK_STARTED emitted synchronously inside + // the first captureFrame is attributed to this segment. + hasCapturedOwnFrame = true; + if (resampler) { for (const f of resampler.push(frame)) { await audioOutput.captureFrame(f); diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index c7d2c0717..5cb3803f9 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -105,6 +105,63 @@ describe('TTS stream idle timeout', () => { expect(audioOut.firstFrameFut.done).toBe(true); }); + it('ignores PLAYBACK_STARTED from another segment before its own first frame', async () => { + // Stalled stream so the forwarder is still waiting on its first read when a + // stray event (from an interrupted overlapping segment) arrives; the idle + // timeout then ends the loop without this segment ever capturing a frame. + const stalledStream = new ReadableStream({ start() {} }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stalledStream, audioOutput, controller, 500); + + // Reject path is expected (no first frame ever captured). + audioOut.firstFrameFut.await.catch(() => {}); + + vi.useFakeTimers(); + + // Stray PLAYBACK_STARTED before this segment captures anything must be ignored. + audioOutput.onPlaybackStarted(Date.now()); + expect(audioOut.firstFrameFut.done).toBe(false); + + const taskPromise = task.result; + await vi.advanceTimersByTimeAsync(600); + await taskPromise; + + vi.useRealTimers(); + + expect(audioOutput.capturedFrames.length).toBe(0); + expect(audioOut.firstFrameFut.rejected).toBe(true); + }); + + it('resamples a rate-mismatched frame even after a stray PLAYBACK_STARTED', async () => { + // Output is 24kHz; frames are 16kHz and must be resampled regardless of any + // stray PLAYBACK_STARTED resolving firstFrameFut early. + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(createSilentFrame(16000)); + controller.enqueue(createSilentFrame(16000)); + controller.close(); + }, + }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + + // Stray event before the loop captures anything must not skip resampling. + audioOutput.onPlaybackStarted(Date.now()); + + await task.result; + + expect(audioOut.firstFrameFut.done).toBe(true); + // Every captured frame must match the output sample rate (i.e. was resampled). + expect(audioOutput.capturedFrames.length).toBeGreaterThan(0); + for (const f of audioOutput.capturedFrames) { + expect(f.sampleRate).toBe(24000); + } + }); + it('performTTSInference completes when TTS node returns stalled stream', async () => { const stalledTtsStream = new ReadableStream({ start(controller) { From b3cea3908b96e0e3e1bfb43b4ab1a02c513cd4c6 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Fri, 12 Jun 2026 17:44:55 +0100 Subject: [PATCH 2/2] clean up comments --- agents/src/voice/generation.ts | 13 ++++--------- agents/src/voice/generation_tts_timeout.test.ts | 3 --- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index a374e219b..05e3555af 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -836,15 +836,10 @@ async function forwardAudio( const reader = ttsStream.getReader(); let resampler: AudioResampler | null = null; - // The audio output is shared across overlapping segments. When a speech is - // interrupted, the main loop immediately authorizes the next speech, so this - // forwarder can register its listener while the interrupted segment's teardown - // is still emitting PLAYBACK_STARTED on the same output. Only honor the event - // once this loop has captured its own first frame, so a stray event from - // another segment can't resolve our `firstFrameFut` prematurely. A premature - // resolution skips resampler creation (gated on `!firstFrameFut.done`) and - // pushes an unresampled frame to the AudioSource, raising - // `RtcError: sample_rate and num_channels don't match`. + // The audio output is shared across overlapping segments, so ignore a + // PLAYBACK_STARTED from another segment until we capture our own first frame. + // Resolving `firstFrameFut` early skips resampler creation and pushes an + // unresampled frame (`RtcError: sample_rate and num_channels don't match`). let hasCapturedOwnFrame = false; const onPlaybackStarted = (ev: { createdAt: number }) => { diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index 5cb3803f9..cbe636ebe 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -120,7 +120,6 @@ describe('TTS stream idle timeout', () => { vi.useFakeTimers(); - // Stray PLAYBACK_STARTED before this segment captures anything must be ignored. audioOutput.onPlaybackStarted(Date.now()); expect(audioOut.firstFrameFut.done).toBe(false); @@ -149,13 +148,11 @@ describe('TTS stream idle timeout', () => { const controller = new AbortController(); const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); - // Stray event before the loop captures anything must not skip resampling. audioOutput.onPlaybackStarted(Date.now()); await task.result; expect(audioOut.firstFrameFut.done).toBe(true); - // Every captured frame must match the output sample rate (i.e. was resampled). expect(audioOutput.capturedFrames.length).toBeGreaterThan(0); for (const f of audioOutput.capturedFrames) { expect(f.sampleRate).toBe(24000);