diff --git a/.changeset/adaptive-interruption-barge-in.md b/.changeset/adaptive-interruption-barge-in.md new file mode 100644 index 000000000..13d7ff8ac --- /dev/null +++ b/.changeset/adaptive-interruption-barge-in.md @@ -0,0 +1,42 @@ +--- +'@livekit/agents': patch +--- + +Make adaptive interruption actually interrupt + +Three defects, each of which on its own could leave a barge-in unheard or un-acted-on. + +**The overlap gate could be disarmed mid-interruption.** `overlapSpeechStarted` is what lets a +user's overlapping audio reach the interruption model, and only a VAD start-of-speech raises it. +Because VAD never re-announces speech already under way, anything that cleared the flag mid-overlap +disarmed the detector for the rest of the agent turn: every remaining frame was dropped, no +inference request was made, and the agent talked straight through the user. Two events did that. A +second speech segment in the same turn (a queued `SpeechHandle`, or the reply after a tool call) +raises `agent-speech-started` again with no `agent-speech-ended` in between, and was treated as a +new turn; an open overlap is now preserved across it, while a genuine new turn still resets the +overlap, audio buffer, cache and counters. And a transport failover rebuilt +`InterruptionStreamBase` from scratch and replayed only `agent-speech-started`, which marks the +agent as speaking but leaves the overlap disarmed; the in-progress overlap is now handed to the +replacement stream through a distinct `agent-speech-resumed` sentinel. A rejected forwarding task +also no longer surfaces as an unhandled rejection during the failover backoff. + +**Ending the pause let one frame of the interrupted speech escape.** `cancelSpeechPause` opens the +audio output's pause gate so the next speech can be admitted, but frames of the speech it has just +interrupted are still parked at that gate and were released before the interrupted reply task +reached its own `clearBuffer()` — 20ms of audio the user had already barged in over. The +interruption is now signalled to the output before the gate opens, so those frames bail instead. + +**A finished interruption silently discarded the next reply.** +`ParticipantAudioOutput.clearBuffer()` resolves an `interruptedFuture` that frames parked at the +pause gate consult to decide whether to bail. Nothing reset that signal until the _next_ segment's +`flush()`, which only runs after that segment's frames have all been captured — so for the whole of +the following reply the signal still described an interruption that was already over. If the output +was paused mid-reply during that window (an ordinary false-interruption pause, on by default), +every remaining frame bailed at the gate and never reached the wire, while the session still +reported the reply as fully spoken and committed it to the chat context. The gate is now scoped to +the segment being captured: a frame bails only for an interruption raised at or after its own +segment began, and parked frames are woken by a per-frame signal so a concurrent `flush()` can no +longer strand one there. + +`OverlappingSpeechEvent` is also now exported by name from `voice/events.js`, so +`overlapping_speech` handlers can be typed without reaching into `inference/interruption/types.js`. diff --git a/.changeset/fix-krisp-frame-identity.md b/.changeset/fix-krisp-frame-identity.md new file mode 100644 index 000000000..36babff8d --- /dev/null +++ b/.changeset/fix-krisp-frame-identity.md @@ -0,0 +1,13 @@ +--- +'@livekit/agents-plugin-krisp': patch +--- + +Fix Krisp-processed audio being invisible to the rest of the pipeline + +The LiveKit Cloud backend is reached through `createRequire`, which resolves the internal +package's `require` condition and so loads the CJS build of `@livekit/rtc-node` next to the +ESM one the framework uses. Frames returned by that backend were instances of the CJS copy's +`AudioFrame`, so every `instanceof AudioFrame` downstream failed. Adaptive interruption saw +zero audio and classified every barge-in as a backchannel, making it impossible to interrupt +an agent that had noise cancellation enabled. Frames are now adopted into the local binding +before leaving the filter, sharing their samples rather than copying them. diff --git a/.changeset/interrupted-reply-backlog-on-the-wire.md b/.changeset/interrupted-reply-backlog-on-the-wire.md new file mode 100644 index 000000000..a4c0e60e5 --- /dev/null +++ b/.changeset/interrupted-reply-backlog-on-the-wire.md @@ -0,0 +1,13 @@ +--- +'@livekit/agents': patch +--- + +Stop an interrupted reply's TTS backlog from reaching the wire + +`captureFrame` only compared a segment's interrupt snapshot while the pause gate was closed, so +the check was skipped entirely once the gate reopened. `cancelSpeechPause` un-gates the sink to +admit the next reply as soon as the handle is interrupted, but the interrupted reply's +`forwardAudio` loop keeps running until its abort signal fires an event loop turn later — and real +TTS providers hand it several seconds of audio ahead of realtime to drain in the meantime. Those +frames took the open-gate path straight to the wire, so the user kept hearing the barged-over +speech resume while the next reply's transcript was already streaming. diff --git a/agents/src/inference/interruption/interruption_pipeline.test.ts b/agents/src/inference/interruption/interruption_pipeline.test.ts new file mode 100644 index 000000000..65a7fb64d --- /dev/null +++ b/agents/src/inference/interruption/interruption_pipeline.test.ts @@ -0,0 +1,431 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// End-to-end (transport-mocked) test of the adaptive-interruption pipeline as it is wired by +// AudioRecognition: room audio -> interruption stream channel -> audio transformer -> WS transport. +// +// Regression coverage for "every overlap is classified as backchannel": the classifier can only +// return `isInterruption: true` from the inference-response path, which requires audio to actually +// reach the transport while the overlap is open. +import { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ChatContext } from '../../llm/chat_context.js'; +import { initializeLogger } from '../../log.js'; +import { AudioRecognition, type RecognitionHooks } from '../../voice/audio_recognition.js'; +import { createEndpointing } from '../../voice/turn_config/endpointing.js'; +import { MockWebSocket } from './_mock_ws.js'; +import { AdaptiveInterruptionDetector } from './interruption_detector.js'; + +vi.mock('ws', async () => { + const { MockWebSocket } = await import('./_mock_ws.js'); + return { default: MockWebSocket, WebSocket: MockWebSocket }; +}); + +initializeLogger({ pretty: false, level: 'silent' }); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitFor(predicate: () => boolean, timeoutMs = 3000): Promise { + const start = performance.now(); + while (!predicate()) { + if (performance.now() - start > timeoutMs) { + throw new Error('condition not met within timeout'); + } + await sleep(5); + } +} + +function createHooks(): RecognitionHooks { + return { + onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), + onStartOfSpeech: vi.fn(), + onVADInferenceDone: vi.fn(), + onEndOfSpeech: vi.fn(), + onInterimTranscript: vi.fn(), + onFinalTranscript: vi.fn(), + onPreemptiveGeneration: vi.fn(), + onAgentBackchannelOpportunity: vi.fn(), + retrieveChatCtx: () => ChatContext.empty(), + onEndOfTurn: vi.fn(async () => true), + } as unknown as RecognitionHooks; +} + +/** 10ms of non-silent mono PCM, as a room track would deliver it. */ +function makeFrame(sampleRate: number): AudioFrame { + const samples = Math.floor(sampleRate / 100); + const data = new Int16Array(samples); + for (let i = 0; i < samples; i++) { + data[i] = Math.round(8000 * Math.sin((2 * Math.PI * 220 * i) / sampleRate)); + } + return new AudioFrame(data, sampleRate, 1, samples); +} + +/** Number of binary (audio) frames the transport pushed onto the socket. */ +function audioSendCount(ws: MockWebSocket): number { + return ws.sent.filter((s) => s instanceof Uint8Array).length; +} + +/** `created_at` header the transport stamped on a binary frame. */ +function createdAtOf(buf: Uint8Array): number { + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + return view.getUint32(4, true) * 0x100000000 + view.getUint32(0, true); +} + +interface Harness { + recognition: AudioRecognition; + detector: AdaptiveInterruptionDetector; + hooks: RecognitionHooks; + ws: MockWebSocket; + events: OverlapEvent[]; + /** Requests the mock gateway has answered so far. */ + requestCount: () => number; + /** Answer the next request with a bargein verdict instead of a plain inference_done. */ + bargeinOnNextRequest: () => void; + close: () => Promise; +} + +/** Audio frames the transport pushed across every socket it has opened so far. */ +function totalAudioSendCount(): number { + return MockWebSocket.instances.reduce((total, ws) => total + audioSendCount(ws), 0); +} + +interface OverlapEvent { + isInterruption: boolean; + numRequests: number; + probability: number; +} + +async function createHarness({ + sampleRate = 48000, +}: { sampleRate?: number } = {}): Promise { + const detector = new AdaptiveInterruptionDetector({ + baseUrl: 'http://localhost:9999', + apiKey: 'test-key', + apiSecret: 'test-secret', + }); + const hooks = createHooks(); + const recognition = new AudioRecognition({ + recognitionHooks: hooks, + interruptionDetection: detector, + endpointing: createEndpointing({ mode: 'dynamic', minDelay: 500, maxDelay: 3000, alpha: 0.9 }), + }); + // The pipeline wiring keys off `interruptionDetection`; the enabled flag additionally requires a + // VAD, which this test substitutes for by driving the overlap callbacks directly. + (recognition as unknown as { isInterruptionEnabled: boolean }).isInterruptionEnabled = true; + + const events: OverlapEvent[] = []; + detector.on('overlapping_speech', (ev) => events.push(ev)); + // A simulated transport failure is emitted as a recoverable detector error; without a listener + // the EventEmitter would rethrow it as an unhandled 'error'. + detector.on('error', () => {}); + + // Continuous room audio, pumped in the background for the whole test, exactly as a subscribed + // track behaves — the interruption stream must pick it up on its own. + let pumping = true; + let pushFrame!: (frame: AudioFrame) => void; + const audioStream = new ReadableStream({ + start(controller) { + pushFrame = (frame) => controller.enqueue(frame); + }, + }); + recognition.setInputAudioStream(audioStream); + const pump = (async () => { + while (pumping) { + pushFrame(makeFrame(sampleRate)); + await sleep(10); + } + })(); + + const ac = new AbortController(); + const task = ( + recognition as unknown as { + createInterruptionTask: ( + d: AdaptiveInterruptionDetector, + signal: AbortSignal, + ) => Promise; + } + ).createInterruptionTask(detector, ac.signal); + + await waitFor(() => MockWebSocket.instances.length > 0); + const ws = MockWebSocket.instances[MockWebSocket.instances.length - 1]!; + ws.simulateOpen(); + await waitFor(() => ws.sent.length > 0); // session.create + ws.simulateMessage({ type: 'session.created', default_threshold: 0.5 }); + await sleep(20); + + // Stand-in for the gateway: answer every audio frame promptly, as the real service does. Without + // this the transport's own 0.7s inference timeout tears the stream down. Sockets opened later + // (an options reconnect, or the replacement stream built by a failover retry) are adopted too, + // so the gateway keeps behaving normally across a reconnect. + const answered = new Map([[ws, 0]]); + let bargeinPending = false; + const responder = (async () => { + while (pumping) { + for (const socket of MockWebSocket.instances) { + if (!answered.has(socket)) { + answered.set(socket, 0); + socket.simulateOpen(); + await sleep(1); + socket.simulateMessage({ type: 'session.created', default_threshold: 0.5 }); + } + const binary = socket.sent.filter((s): s is Uint8Array => s instanceof Uint8Array); + let seen = answered.get(socket)!; + while (seen < binary.length) { + const createdAt = createdAtOf(binary[seen]!); + seen++; + if (bargeinPending) { + bargeinPending = false; + socket.simulateMessage({ + type: 'bargein_detected', + created_at: createdAt, + probabilities: [0.91, 0.93, 0.95], + prediction_duration: 0.02, + }); + } else { + socket.simulateMessage({ + type: 'inference_done', + created_at: createdAt, + probabilities: [0.01, 0.02], + prediction_duration: 0.02, + is_bargein: false, + }); + } + } + answered.set(socket, seen); + } + await sleep(5); + } + })(); + + return { + recognition, + detector, + hooks, + ws, + events, + requestCount: () => [...answered.values()].reduce((a, b) => a + b, 0), + bargeinOnNextRequest: () => { + bargeinPending = true; + }, + close: async () => { + pumping = false; + await pump; + await responder; + ac.abort(); + await task.catch(() => {}); + await recognition.close().catch(() => {}); + }, + }; +} + +describe('adaptive interruption pipeline', () => { + beforeEach(() => { + MockWebSocket.instances.length = 0; + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('sends inference requests for user audio that overlaps agent speech', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(600); + + expect(audioSendCount(h.ws)).toBeGreaterThan(0); + + await h.close(); + }); + + it('classifies a server bargein verdict as an interruption', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + h.bargeinOnNextRequest(); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(600); + + expect(audioSendCount(h.ws)).toBeGreaterThan(0); + await waitFor(() => h.events.length > 0); + + expect(h.events[0]!.isInterruption).toBe(true); + await waitFor(() => vi.mocked(h.hooks.onInterruption).mock.calls.length > 0); + + await h.close(); + }); + + it('keeps making requests across repeated short overlaps in one agent turn', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(300); + + const perOverlapRequests: number[] = []; + for (let i = 0; i < 3; i++) { + const before = audioSendCount(h.ws); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(1200); // the user's overlaps were 1-2s each + await h.recognition.onEndOfOverlapSpeech(Date.now()); + await sleep(50); + perOverlapRequests.push(audioSendCount(h.ws) - before); + } + + expect(h.events).toHaveLength(3); + expect(perOverlapRequests.every((n) => n > 0)).toBe(true); + expect(h.events.map((e) => e.numRequests).every((n) => n > 0)).toBe(true); + + await h.close(); + }); +}); + +// --------------------------------------------------------------------------- +// Overlap state must survive events that are not a new agent turn (regression) +// --------------------------------------------------------------------------- + +/** + * `overlapSpeechStarted` is the gate that lets user audio reach the gateway at all, and the only + * thing that ever raises it is an `overlap-speech-started` sentinel, which in turn only comes from + * a VAD start-of-speech. VAD does not re-announce speech that is already under way, so once the + * flag is cleared mid-overlap nothing can re-arm it: every remaining frame of the interruption is + * dropped, no inference request is made, and the agent talks straight through the user. + * + * Two things clear the flag without the user's turn having ended. + */ +describe('adaptive interruption overlap state retention', () => { + beforeEach(() => { + MockWebSocket.instances.length = 0; + }); + + /** + * A transient socket failure fails the stream over. JS rebuilds `InterruptionStreamBase` from + * scratch on retry — all of its state lives in a `setupTransform()` closure — whereas Python + * keeps `_agent_speech_started` / `_overlap_started` on `self` and only reconnects the socket. + * Replaying `agent-speech-started` alone restores half the state: the agent is known to be + * speaking again, but the in-flight overlap is gone. + */ + it('keeps sending user audio after the transport fails over mid-overlap', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(400); + + expect(totalAudioSendCount()).toBeGreaterThan(0); + const socketsBefore = MockWebSocket.instances.length; + + // Transient socket failure while the user is mid-interrupt. + h.ws.emit('error', new Error('connection reset')); + + // The failover sleeps intervalForRetry(0) (2s + jitter) before rebuilding the stream. + await waitFor(() => MockWebSocket.instances.length > socketsBefore, 8000); + await sleep(200); + + // The user is still talking; from here on every frame must reach the new socket. + const sendsAfterFailover = totalAudioSendCount(); + await sleep(800); + + expect(totalAudioSendCount() - sendsAfterFailover).toBeGreaterThan(0); + + await h.recognition.onEndOfOverlapSpeech(Date.now()); + await waitFor(() => h.events.length > 0); + expect(h.events[h.events.length - 1]!.numRequests).toBeGreaterThan(0); + + await h.close(); + }, 30_000); + + /** + * One user-perceived agent turn can contain several speech segments — a tool call sandwiched + * between two replies, or a queued `say()`. `AgentActivity.onPipelineReplyDone` only reports + * `onEndOfAgentSpeech` once the speech queue has drained, so the second segment raises + * `agent-speech-started` again with no `agent-speech-ended` in between. Treating that as a new + * turn resets the overlap the user is in the middle of. + */ + it('keeps sending user audio when a second speech segment starts mid-overlap', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(400); + + const before = totalAudioSendCount(); + expect(before).toBeGreaterThan(0); + + // Second segment of the same turn: no `onEndOfAgentSpeech` precedes it. + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(600); + + expect(totalAudioSendCount() - before).toBeGreaterThan(0); + + await h.recognition.onEndOfOverlapSpeech(Date.now()); + await waitFor(() => h.events.length > 0); + expect(h.events[h.events.length - 1]!.numRequests).toBeGreaterThan(0); + + await h.close(); + }, 30_000); + + /** + * The bargein verdict must still surface after a mid-overlap segment change — restoring the gate + * is only useful if the recovered audio can still produce `isInterruption: true`. + */ + it('still reports a bargein after a second speech segment starts mid-overlap', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(400); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + h.bargeinOnNextRequest(); + await waitFor(() => h.events.length > 0, 5000); + + expect(h.events[0]!.isInterruption).toBe(true); + expect(h.events[0]!.numRequests).toBeGreaterThan(0); + await waitFor(() => vi.mocked(h.hooks.onInterruption).mock.calls.length > 0); + + await h.close(); + }, 30_000); + + /** A genuine new turn must still wipe the overlap, the cache and the counters. */ + it('resets overlap state on a new agent turn', async () => { + const h = await createHarness(); + + await h.recognition.onStartOfAgentSpeech(Date.now()); + await sleep(200); + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(400); + expect(totalAudioSendCount()).toBeGreaterThan(0); + + // The agent turn ends, then a new one begins. The user is no longer overlapping anything, + // so their audio must not be forwarded until a fresh overlap is announced. + await h.recognition.onEndOfAgentSpeech(Date.now()); + await sleep(20); + await h.recognition.onStartOfAgentSpeech(Date.now()); + + const afterNewTurn = totalAudioSendCount(); + await sleep(600); + expect(totalAudioSendCount()).toBe(afterNewTurn); + + // ...and the new turn's first overlap starts from a clean counter. + await h.recognition.onStartOfOverlapSpeech(200, Date.now()); + await sleep(400); + await h.recognition.onEndOfOverlapSpeech(Date.now()); + await waitFor(() => h.events.length > 0); + + const last = h.events[h.events.length - 1]!; + expect(last.numRequests).toBeGreaterThan(0); + expect(totalAudioSendCount()).toBeGreaterThan(afterNewTurn); + + await h.close(); + }, 30_000); +}); diff --git a/agents/src/inference/interruption/interruption_stream.ts b/agents/src/inference/interruption/interruption_stream.ts index efb02819f..ee41c3ef9 100644 --- a/agents/src/inference/interruption/interruption_stream.ts +++ b/agents/src/inference/interruption/interruption_stream.ts @@ -14,6 +14,7 @@ import { InterruptionCacheEntry } from './interruption_cache_entry.js'; import type { AdaptiveInterruptionDetector } from './interruption_detector.js'; import { type AgentSpeechEnded, + type AgentSpeechResumed, type AgentSpeechStarted, type ApiConnectOptions, type Flush, @@ -29,6 +30,7 @@ import { createWsTransport } from './ws_transport.js'; // Re-export sentinel types for backwards compatibility export type { AgentSpeechEnded, + AgentSpeechResumed, AgentSpeechStarted, ApiConnectOptions, Flush, @@ -37,6 +39,12 @@ export type { OverlapSpeechStarted, }; +/** Snapshot of an overlap that is still being classified. */ +export interface ActiveOverlap { + startedAt: number; + userSpeakingSpan?: Span; +} + export class InterruptionStreamSentinel { static agentSpeechStarted(): AgentSpeechStarted { return { type: 'agent-speech-started' }; @@ -46,6 +54,14 @@ export class InterruptionStreamSentinel { return { type: 'agent-speech-ended' }; } + static agentSpeechResumed(overlap?: ActiveOverlap): AgentSpeechResumed { + return { + type: 'agent-speech-resumed', + overlapStartedAt: overlap?.startedAt, + userSpeakingSpan: overlap?.userSpeakingSpan, + }; + } + static overlapSpeechStarted( speechDuration: number, startedAt: number, @@ -100,6 +116,9 @@ export class InterruptionStreamBase { private wsClose?: () => void; + // The overlap flag lives in the setupTransform() closure; this exposes it for `activeOverlap`. + private readOverlapSpeechStarted?: () => boolean; + // Mutable transport options that can be updated via updateOptions() private transportOptions: { baseUrl: string; @@ -139,6 +158,21 @@ export class InterruptionStreamBase { this.eventStream = this.setupTransform(); } + /** + * The overlap this stream is currently classifying, if any. + * + * Unlike Python — where the equivalent flags are attributes on the stream instance and a + * reconnect leaves them alone — every retry here builds a new stream and drops the state held in + * `setupTransform()`. Callers that recreate the stream use this to hand the in-progress overlap + * to its replacement via {@link InterruptionStreamSentinel.agentSpeechResumed}. + */ + get activeOverlap(): ActiveOverlap | undefined { + if (!this.readOverlapSpeechStarted?.() || this.overlapSpeechStartedAt === undefined) { + return undefined; + } + return { startedAt: this.overlapSpeechStartedAt, userSpeakingSpan: this.userSpeakingSpan }; + } + /** * Update stream options. For WebSocket transport, this triggers a reconnection. */ @@ -184,6 +218,7 @@ export class InterruptionStreamBase { overlapSpeechStarted = partial.overlapSpeechStarted; } }; + this.readOverlapSpeechStarted = () => overlapSpeechStarted; const handleSpanUpdate = (entry: InterruptionCacheEntry) => { if (this.userSpeakingSpan) { updateUserSpeakingSpan(this.userSpeakingSpan, entry); @@ -236,15 +271,41 @@ export class InterruptionStreamBase { controller.enqueue(audioSlice); } } else if (chunk.type === 'agent-speech-started') { - this.logger.debug('agent speech started'); + // One agent turn can span several speech segments — a queued SpeechHandle, or the + // reply that follows a tool call — and `AgentActivity.onPipelineReplyDone` only + // reports `agent-speech-ended` once the speech queue has drained. The later segments + // therefore arrive here with no end in between. Resetting on those would strand an + // overlap the user is still in the middle of: `overlapSpeechStarted` is the gate that + // lets their audio reach the gateway at all, and only a VAD start-of-speech can raise + // it again — which never comes for speech that is already under way. + if (agentSpeechStarted && overlapSpeechStarted) { + this.logger.debug('agent speech continued into a new segment, keeping open overlap'); + } else { + this.logger.debug('agent speech started'); + agentSpeechStarted = true; + overlapSpeechStarted = false; + this.overlapSpeechStartedAt = undefined; + accumulatedSamples = 0; + overlapCount = 0; + startIdx = 0; + this.numRequests = 0; + cache.clear(); + } + } else if (chunk.type === 'agent-speech-resumed') { + // This stream replaces one the transport failover tore down. Adopt what the previous + // stream knew instead of treating it as a new turn; everything else is already at its + // freshly-constructed value. + this.logger.debug( + { overlapStartedAt: chunk.overlapStartedAt }, + 'resuming agent speech on a replacement interruption stream', + ); agentSpeechStarted = true; - overlapSpeechStarted = false; - this.overlapSpeechStartedAt = undefined; - accumulatedSamples = 0; - overlapCount = 0; - startIdx = 0; - this.numRequests = 0; - cache.clear(); + if (chunk.overlapStartedAt !== undefined) { + overlapSpeechStarted = true; + overlapCount = 1; + this.overlapSpeechStartedAt = chunk.overlapStartedAt; + this.userSpeakingSpan = chunk.userSpeakingSpan; + } } else if (chunk.type === 'agent-speech-ended') { this.logger.debug('agent speech ended'); agentSpeechStarted = false; diff --git a/agents/src/inference/interruption/types.ts b/agents/src/inference/interruption/types.ts index d062eb739..38ec96a23 100644 --- a/agents/src/inference/interruption/types.ts +++ b/agents/src/inference/interruption/types.ts @@ -59,6 +59,18 @@ export interface AgentSpeechEnded { type: 'agent-speech-ended'; } +/** + * Restores a replacement stream after the transport failed over, which builds a brand new + * `InterruptionStreamBase` with none of the previous one's state. Kept distinct from + * {@link AgentSpeechStarted} so that sentinel keeps meaning "a new agent turn began, reset". + */ +export interface AgentSpeechResumed { + type: 'agent-speech-resumed'; + /** Absolute timestamp (ms) the overlap started at, when one was still open at failover. */ + overlapStartedAt?: number; + userSpeakingSpan?: Span; +} + export interface OverlapSpeechStarted { type: 'overlap-speech-started'; /** Duration of the speech segment in milliseconds (matches VADEvent.speechDuration units). */ @@ -86,6 +98,7 @@ export interface Flush { export type InterruptionSentinel = | AgentSpeechStarted | AgentSpeechEnded + | AgentSpeechResumed | OverlapSpeechStarted | OverlapSpeechEnded | Flush; diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index 3b3ed17e2..9bd2247f5 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -408,6 +408,79 @@ describe('AgentActivity - mainTask', () => { expect(handle.interrupted).toBe(true); expect(fakeActivity.pausedSpeech).toBeUndefined(); }); + + /** + * Unit-level guard for the ordering measured end to end in + * `confirmed_interruption_pause_and_commit.test.ts`. It pins the cheap invariant that test cannot: + * the sink is only cleared when this call is what interrupted the paused speech. + */ + it('clears the sink before un-gating it, and only when it interrupted the paused speech', async () => { + const makeFixture = (handle: SpeechHandle) => { + const calls: string[] = []; + const audioOutput = { + canPause: true, + pause: vi.fn(() => { + calls.push('pause'); + }), + resume: vi.fn(() => { + calls.push('resume'); + }), + clearBuffer: vi.fn(() => { + calls.push('clearBuffer'); + }), + }; + const fakeActivity = { + cancelSpeechPauseTask: undefined as Promise | undefined, + falseInterruptionTimer: undefined as NodeJS.Timeout | undefined, + pausedSpeech: { handle, agentState: 'speaking', timeout: 2000 } as + | { handle: SpeechHandle; agentState: string; timeout: number } + | undefined, + _currentSpeech: handle, + logger: { debug: vi.fn(), info: vi.fn() }, + agentSession: { + sessionOptions: { + turnHandling: { + interruption: { resumeFalseInterruption: true, falseInterruptionTimeout: 2000 }, + }, + }, + output: { audio: audioOutput }, + }, + }; + const proto = AgentActivity.prototype as unknown as Record< + string, + (...args: never[]) => unknown + >; + return { + calls, + fakeActivity, + cancelSpeechPause: proto['cancelSpeechPause']!.bind(fakeActivity as never) as (options?: { + interrupt?: boolean; + }) => Promise, + }; + }; + + const interrupting = makeFixture( + (() => { + const h = SpeechHandle.create({ allowInterruptions: true }); + h._authorizeGeneration(); + return h; + })(), + ); + await raceTimeout(interrupting.cancelSpeechPause(), 2000); + expect(interrupting.calls).toEqual(['clearBuffer', 'resume']); + + // `interrupt: false` ends the pause without interrupting — the speech is meant to keep + // playing, so clearing its buffer would throw away audio that is still wanted. + const resuming = makeFixture( + (() => { + const h = SpeechHandle.create({ allowInterruptions: true }); + h._authorizeGeneration(); + return h; + })(), + ); + await raceTimeout(resuming.cancelSpeechPause({ interrupt: false }), 2000); + expect(resuming.calls).toEqual(['resume']); + }); }); describe('AgentActivity - speech completion', () => { diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 23d8b4ea9..95b7ca14c 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -4586,12 +4586,14 @@ export class AgentActivity implements RecognitionHooks { return; } + let interruptedPausedSpeech = false; if ( interrupt && !this.pausedSpeech.handle.interrupted && this.pausedSpeech.handle.allowInterruptions ) { this.pausedSpeech.handle.interrupt(); + interruptedPausedSpeech = true; // ensure the generation is done — but only if a generation // was actually started. Must be raced against interrupt: an interrupted // paused speech may never mark its generation done, and an un-raced @@ -4606,6 +4608,15 @@ export class AgentActivity implements RecognitionHooks { const interruptionOptions = this.agentSession.sessionOptions.turnHandling.interruption; if (interruptionOptions.resumeFalseInterruption && this.agentSession.output.audio) { + // Frames of the speech just interrupted are parked at the sink's pause gate. Opening the + // gate is only meant to admit the *next* speech, but the parked frames are released first + // and audio the user has already barged in over reaches the wire. Python blocks here until + // the generation finishes, which gets it the same ordering; the await above is raced + // against the interrupt (#1124) and so returns immediately, before the interrupted reply + // task has run its own clearBuffer(). Signal the interruption first instead. + if (interruptedPausedSpeech) { + this.agentSession.output.audio.clearBuffer(); + } this.agentSession.output.audio.resume(); } } diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index d11ed4680..8f6fc157c 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -23,7 +23,10 @@ import { import { apiConnectDefaults, intervalForRetry } from '../inference/interruption/defaults.js'; import { InterruptionDetectionError } from '../inference/interruption/errors.js'; import type { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js'; -import { InterruptionStreamSentinel } from '../inference/interruption/interruption_stream.js'; +import { + type ActiveOverlap, + InterruptionStreamSentinel, +} from '../inference/interruption/interruption_stream.js'; import { type InterruptionSentinel, type OverlappingSpeechEvent, @@ -1960,6 +1963,8 @@ export class AudioRecognition { let numRetries = 0; const maxRetries = apiConnectDefaults.maxRetries; + // Overlap the torn-down stream was still classifying, handed to its replacement below. + let resumeOverlap: ActiveOverlap | undefined; while (!signal.aborted) { const stream = interruptionDetection.createStream(); @@ -1980,11 +1985,14 @@ export class AudioRecognition { let forwardTask: Promise | undefined; try { - // Unlike Python where _agent_speech_started lives on `self` and survives retries, - // JS creates a fresh InterruptionStreamBase per retry with agentSpeechStarted = false. - // Re-inject the sentinel so the new stream knows the agent is mid-speech. + // Unlike Python where _agent_speech_started and _overlap_started live on `self` and are + // untouched by a reconnect, JS creates a fresh InterruptionStreamBase per retry and loses + // everything the previous one held. Hand it the whole picture, not just "the agent is + // speaking": an overlap that was open at failover can never be re-armed otherwise, since + // only a VAD start-of-speech raises the flag and VAD does not re-announce speech that is + // already under way. The rest of that turn's user audio would be dropped on the floor. if (numRetries > 0 && this.isAgentSpeaking) { - await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted()); + await stream.pushFrame(InterruptionStreamSentinel.agentSpeechResumed(resumeOverlap)); } forwardTask = (async () => { @@ -2005,6 +2013,10 @@ export class AudioRecognition { inputReader.releaseLock(); } })(); + // The stream this task pushes into is torn down as soon as the read loop below fails, so + // it rejects long before the `finally` attaches a handler — which only runs after the + // retry backoff. Mark it handled now so the rejection is not reported as unhandled. + forwardTask.catch(() => {}); const abortPromise = waitForAbort(signal); @@ -2069,6 +2081,8 @@ export class AudioRecognition { break; } } finally { + // Read before cleanup() closes the stream, so a retry can restore the open overlap. + resumeOverlap = stream.activeOverlap; await cleanup(); await forwardTask?.catch((e) => { this.logger.debug({ err: e }, 'interruption task exited with error'); diff --git a/agents/src/voice/confirmed_interruption_pause_and_commit.test.ts b/agents/src/voice/confirmed_interruption_pause_and_commit.test.ts new file mode 100644 index 000000000..9801fddac --- /dev/null +++ b/agents/src/voice/confirmed_interruption_pause_and_commit.test.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame, type Room, TrackPublishOptions, TrackSource } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { describe, expect, it } from 'vitest'; +import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; +import { initializeLogger } from '../log.js'; +import { VADEventType } from '../vad.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes } from './events.js'; +import { ParticipantAudioOutput } from './room_io/_output.js'; +import { FakeLLM } from './testing/fake_llm.js'; + +const SAMPLE_RATE = 24000; +const FRAME_MS = 20; +const FRAMES_PER_REPLY = 40; +const FALSE_INTERRUPTION_TIMEOUT = 400; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function frame(): AudioFrame { + const samples = (SAMPLE_RATE * FRAME_MS) / 1000; + return new AudioFrame(new Int16Array(samples), SAMPLE_RATE, 1, samples); +} + +function vadEvent(type: VADEventType, speechDuration = 0, silenceDuration = 0) { + return { + type, + samplesIndex: 0, + timestamp: Date.now(), + speechDuration, + silenceDuration, + frames: [], + probability: 1, + inferenceDuration: 0, + speaking: type === VADEventType.START_OF_SPEECH, + rawAccumulatedSilence: 0, + rawAccumulatedSpeech: 0, + } as never; +} + +function sttFinal(text: string) { + return { + type: 'final_transcript', + alternatives: [{ text, language: 'en', startTime: 0, endTime: 0, confidence: 1 }], + } as never; +} + +/** The verdict shape `AudioRecognition` forwards on `bargein_detected`. */ +function bargeInVerdict(overlapStartedAt: number): OverlappingSpeechEvent { + return { + type: 'overlapping_speech', + detectedAt: Date.now(), + isInterruption: true, + overlapStartedAt, + totalDurationInS: 0.1, + predictionDurationInS: 0.05, + detectionDelayInS: 0.2, + probability: 0.99, + numRequests: 1, + }; +} + +/** The real ParticipantAudioOutput; only track publishing is skipped (no LiveKit server here). */ +class TestParticipantAudioOutput extends ParticipantAudioOutput { + constructor() { + super({} as Room, { + sampleRate: SAMPLE_RATE, + numChannels: 1, + trackPublishOptions: new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }), + queueSizeMs: 100_000, + }); + (this as unknown as { startedFuture: { resolve: () => void } }).startedFuture.resolve(); + } + + override async start(): Promise {} +} + +/** Emits frames spaced out in time so a barge-in can land mid-reply. */ +class FrameAgent extends Agent { + produced = 0; + onFrame?: (produced: number) => void; + + constructor() { + super({ instructions: 'test' }); + } + + async ttsNode(): Promise> { + let emitted = 0; + return new ReadableStream({ + pull: async (controller) => { + if (emitted >= FRAMES_PER_REPLY) { + controller.close(); + return; + } + if (emitted > 0) await sleep(15); + emitted++; + controller.enqueue(frame()); + this.produced++; + this.onFrame?.(this.produced); + }, + }); + } +} + +async function makeHarness() { + const session = new AgentSession({ + llm: new FakeLLM([{ input: 'one', content: 'first reply' }]), + aecWarmupDuration: null, + turnHandling: { interruption: { falseInterruptionTimeout: FALSE_INTERRUPTION_TIMEOUT } }, + }); + + const out = new TestParticipantAudioOutput(); + session.output.audio = out; + + // Frame accounting at the boundary that matters: how many frames actually reached the + // LiveKit AudioSource, i.e. the wire. + const source = ( + out as unknown as { audioSource: { captureFrame: (f: AudioFrame) => Promise } } + ).audioSource; + const captureFrame = source.captureFrame.bind(source); + let delivered = 0; + source.captureFrame = async (f: AudioFrame) => { + delivered++; + return captureFrame(f); + }; + + const falseInterruptions: boolean[] = []; + session.on(AgentSessionEventTypes.AgentFalseInterruption, (ev) => + falseInterruptions.push(ev.resumed), + ); + + const agent = new FrameAgent(); + await session.start({ agent }); + + const gate = out as unknown as { playbackEnabledFuture: { done: boolean } }; + + const waitForProduced = (n: number) => + new Promise((resolve) => { + agent.onFrame = (produced) => { + if (produced >= n) resolve(); + }; + }); + + return { + session, + agent, + falseInterruptions, + delivered: () => delivered, + paused: () => !gate.done, + waitForProduced, + activity: () => session._activity!, + async close() { + await session.close(); + await out.close(); + }, + }; +} + +type Harness = Awaited>; + +/** + * Drives a reply up to the moment the adaptive interruption model rules the overlap a genuine + * barge-in, then lets the user fall silent so the false-interruption timer is armed. + * + * Returns the frame count at the AudioSource taken just before the verdict, so callers can + * measure what reaches the wire after it. + */ +async function bargeInMidReply(h: Harness) { + const handle = h.session.generateReply({ userInput: 'one' }); + await h.waitForProduced(3); + + // The user starts talking over the agent: VAD parks the reply at the pause gate. + const overlapStartedAt = Date.now(); + h.activity().onStartOfSpeech(vadEvent(VADEventType.START_OF_SPEECH)); + h.activity().onVADInferenceDone(vadEvent(VADEventType.INFERENCE_DONE, 600)); + expect(h.paused()).toBe(true); + + // Let the TTS keep producing so frames are genuinely parked at the gate when the verdict + // lands — those parked frames are the ones that can escape. + await sleep(60); + + const deliveredBeforeVerdict = h.delivered(); + h.activity().onInterruption(bargeInVerdict(overlapStartedAt)); + // The user stops talking, which is what arms the false-interruption timer. + h.activity().onEndOfSpeech(vadEvent(VADEventType.END_OF_SPEECH, 600, 200)); + + return { handle, deliveredBeforeVerdict }; +} + +describe('confirmed barge-in: pause, then commit on the transcript', () => { + initializeLogger({ pretty: false, level: 'silent' }); + + /** + * Parity with Python's `AgentActivity.on_interruption` (`agent_activity.py`), which does + * restore → interrupt-by-audio-activity → `_on_end_of_agent_speech` and stops. The model's + * verdict alone only *pauses* the reply; committing the user's turn needs an STT final + * transcript. When none arrives within `falseInterruptionTimeout`, the false-interruption + * timer puts the speech back rather than leaving the user in dead air, and the session + * reports that with `agent_false_interruption(resumed: true)`. + */ + it('resumes the paused reply when no transcript follows the verdict', async () => { + const h = await makeHarness(); + try { + const { handle, deliveredBeforeVerdict } = await bargeInMidReply(h); + + await sleep(FALSE_INTERRUPTION_TIMEOUT + 600); + await handle.waitForPlayout(); + + // The verdict did not commit the turn, so the reply resumes where it stopped. + expect(handle.interrupted).toBe(false); + expect(h.falseInterruptions).toEqual([true]); + expect(h.delivered() - deliveredBeforeVerdict).toBeGreaterThan(0); + expect(h.delivered()).toBe(FRAMES_PER_REPLY); + } finally { + await h.close(); + } + }, 30000); + + /** + * The other half of the same flow, and the one the sink ordering in `cancelSpeechPause()` + * exists for. The final transcript arrives, so `onFinalTranscript` commits the interruption + * through `cancelSpeechPause()`, which un-gates the output so the *next* speech can be + * admitted. Frames of the reply just interrupted are still parked at that gate; unless the + * interruption is signalled to the sink first, they are released before the interrupted + * reply task reaches its own `clearBuffer()` and audio the user has already barged in over + * reaches the wire. + * + * Measured at the AudioSource boundary, not by asserting a call order. + */ + it('delivers no further audio once the final transcript commits the barge-in', async () => { + const h = await makeHarness(); + try { + const { handle, deliveredBeforeVerdict } = await bargeInMidReply(h); + + // STT finalizes what the user said, well inside the false-interruption timeout. + h.activity().onFinalTranscript(sttFinal('stop please'), false); + + await handle.waitForPlayout(); + // Well past the timeout: nothing may resume the speech either. + await sleep(FALSE_INTERRUPTION_TIMEOUT + 300); + + expect(h.delivered() - deliveredBeforeVerdict).toBe(0); + expect(handle.interrupted).toBe(true); + expect(h.falseInterruptions).toEqual([]); + // The reply was cut short: this is a barge-in, not a completed turn. + expect(h.delivered()).toBeLessThan(FRAMES_PER_REPLY); + } finally { + await h.close(); + } + }, 30000); +}); diff --git a/agents/src/voice/events.ts b/agents/src/voice/events.ts index 3a3050f4b..45d19489b 100644 --- a/agents/src/voice/events.ts +++ b/agents/src/voice/events.ts @@ -436,6 +436,12 @@ export const createAgentFalseInterruptionEvent = ({ createdAt, }); +/** + * Payload of {@link AgentSessionEventTypes.OverlappingSpeech}. Re-exported here so handlers can be + * typed by name; it carries `probability`, `probabilities` and `numRequests` alongside the verdict. + */ +export type { OverlappingSpeechEvent }; + export type AgentEvent = | UserInputTranscribedEvent | UserStateChangedEvent diff --git a/agents/src/voice/false_interruption_audio_loss.test.ts b/agents/src/voice/false_interruption_audio_loss.test.ts new file mode 100644 index 000000000..f4432fd01 --- /dev/null +++ b/agents/src/voice/false_interruption_audio_loss.test.ts @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame, type Room, TrackPublishOptions, TrackSource } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { describe, expect, it } from 'vitest'; +import { initializeLogger } from '../log.js'; +import { VADEventType } from '../vad.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes } from './events.js'; +import { ParticipantAudioOutput } from './room_io/_output.js'; +import { FakeLLM } from './testing/fake_llm.js'; + +const SAMPLE_RATE = 24000; +const FRAME_MS = 20; +const FRAMES_PER_REPLY = 20; +const FALSE_INTERRUPTION_TIMEOUT = 400; + +function frame(): AudioFrame { + const samples = (SAMPLE_RATE * FRAME_MS) / 1000; + return new AudioFrame(new Int16Array(samples), SAMPLE_RATE, 1, samples); +} + +function vadEvent(type: VADEventType, speechDuration = 0, silenceDuration = 0) { + return { + type, + samplesIndex: 0, + timestamp: Date.now(), + speechDuration, + silenceDuration, + frames: [], + probability: 1, + inferenceDuration: 0, + speaking: type === VADEventType.START_OF_SPEECH, + rawAccumulatedSilence: 0, + rawAccumulatedSpeech: 0, + } as never; +} + +function sttFinal(text: string) { + return { + type: 'final_transcript', + alternatives: [{ text, language: 'en', startTime: 0, endTime: 0, confidence: 1 }], + } as never; +} + +/** The real ParticipantAudioOutput; only track publishing is skipped (no LiveKit server here). */ +class TestParticipantAudioOutput extends ParticipantAudioOutput { + constructor() { + super({} as Room, { + sampleRate: SAMPLE_RATE, + numChannels: 1, + trackPublishOptions: new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }), + queueSizeMs: 100_000, + }); + (this as unknown as { startedFuture: { resolve: () => void } }).startedFuture.resolve(); + } + + override async start(): Promise {} +} + +/** Emits `FRAMES_PER_REPLY` frames spaced out in time so a turn can be interleaved with VAD events. */ +class FrameAgent extends Agent { + produced = 0; + onFrame?: (produced: number) => void; + + constructor() { + super({ instructions: 'test' }); + } + + async ttsNode(): Promise> { + let emitted = 0; + const emit = () => { + this.produced++; + this.onFrame?.(this.produced); + }; + return new ReadableStream({ + pull: async (controller) => { + if (emitted >= FRAMES_PER_REPLY) { + controller.close(); + return; + } + if (emitted > 0) await new Promise((resolve) => setTimeout(resolve, 15)); + emitted++; + controller.enqueue(frame()); + emit(); + }, + }); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function makeHarness() { + const session = new AgentSession({ + llm: new FakeLLM([ + { input: 'one', content: 'first reply' }, + { input: 'two', content: 'second reply' }, + ]), + aecWarmupDuration: null, + turnHandling: { interruption: { falseInterruptionTimeout: FALSE_INTERRUPTION_TIMEOUT } }, + }); + + const out = new TestParticipantAudioOutput(); + session.output.audio = out; + + // Frame accounting at the boundary the bug lives on: how many frames actually reached + // the LiveKit AudioSource, i.e. the wire. + const source = ( + out as unknown as { audioSource: { captureFrame: (f: AudioFrame) => Promise } } + ).audioSource; + const captureFrame = source.captureFrame.bind(source); + let delivered = 0; + source.captureFrame = async (f: AudioFrame) => { + delivered++; + return captureFrame(f); + }; + + const falseInterruptions: boolean[] = []; + session.on(AgentSessionEventTypes.AgentFalseInterruption, (ev) => + falseInterruptions.push(ev.resumed), + ); + + const agent = new FrameAgent(); + await session.start({ agent }); + + const gate = out as unknown as { playbackEnabledFuture: { done: boolean } }; + + const waitForProduced = (n: number) => + new Promise((resolve) => { + agent.onFrame = (produced) => { + if (produced >= n) resolve(); + }; + }); + + return { + session, + out, + agent, + falseInterruptions, + delivered: () => delivered, + paused: () => !gate.done, + waitForProduced, + activity: () => session._activity!, + async close() { + await session.close(); + await out.close(); + }, + }; +} + +type Harness = Awaited>; + +/** + * One reply that a backchannel pauses mid-flight and the false-interruption timer resumes. + * Every frame the TTS produced must reach the wire: the ones captured while the output was + * paused are held at the gate and pushed on resume. + */ +async function falseInterruptedReply(h: Harness, userInput: string) { + const producedBefore = h.agent.produced; + const deliveredBefore = h.delivered(); + + const handle = h.session.generateReply({ userInput }); + await h.waitForProduced(producedBefore + 3); + + h.activity().onStartOfSpeech(vadEvent(VADEventType.START_OF_SPEECH)); + h.activity().onVADInferenceDone(vadEvent(VADEventType.INFERENCE_DONE, 600)); + const pausedMidReply = h.paused(); + // The user only said "mm-hmm": no final transcript follows, so the false-interruption + // timer is what resumes playback. + h.activity().onEndOfSpeech(vadEvent(VADEventType.END_OF_SPEECH, 0, 100)); + + await handle.waitForPlayout(); + await sleep(FALSE_INTERRUPTION_TIMEOUT + 300); + + return { + pausedMidReply, + produced: h.agent.produced - producedBefore, + delivered: h.delivered() - deliveredBefore, + }; +} + +describe('false interruption after a previous interruption', () => { + initializeLogger({ pretty: false, level: 'silent' }); + + it('delivers the whole reply when nothing was interrupted before', async () => { + const h = await makeHarness(); + try { + const reply = await falseInterruptedReply(h, 'one'); + + expect(reply.pausedMidReply).toBe(true); + expect(reply.produced).toBe(FRAMES_PER_REPLY); + expect(reply.delivered).toBe(reply.produced); + expect(h.falseInterruptions).toEqual([true]); + } finally { + await h.close(); + } + }, 30000); + + it('delivers the whole reply when the previous turn ended in a barge-in', async () => { + const h = await makeHarness(); + try { + // A genuine barge-in: VAD pauses the output, the final transcript confirms the + // interruption, and the reply task calls clearBuffer() on the way out. That leaves + // interruptedFuture resolved, and nothing resets it until the *next* segment's flush — + // which only happens after that segment's frames have all been captured. + const bargedInto = h.session.generateReply({ userInput: 'one' }); + await h.waitForProduced(3); + h.activity().onStartOfSpeech(vadEvent(VADEventType.START_OF_SPEECH)); + h.activity().onVADInferenceDone(vadEvent(VADEventType.INFERENCE_DONE, 600)); + h.activity().onFinalTranscript(sttFinal('stop please'), false); + await bargedInto.waitForPlayout(); + await sleep(150); + + // An ordinary false interruption on the next reply. Every frame captured during the + // pause used to bail at the gate on the stale signal, losing the rest of the reply + // while the session still recorded it as fully spoken. + const reply = await falseInterruptedReply(h, 'two'); + + expect(reply.pausedMidReply).toBe(true); + expect(reply.produced).toBe(FRAMES_PER_REPLY); + expect(reply.delivered).toBe(reply.produced); + expect(h.falseInterruptions).toEqual([true]); + } finally { + await h.close(); + } + }, 30000); + + it('delivers the whole reply when a paused reply completed and was then cancelled', async () => { + const h = await makeHarness(); + try { + // The agent finishes its sentence while the user talks over the tail: all frames are + // captured, then the output is paused and the segment drains to a clean finish. + const completed = h.session.generateReply({ userInput: 'one' }); + await h.waitForProduced(FRAMES_PER_REPLY); + for (let i = 0; i < 200 && h.delivered() < FRAMES_PER_REPLY; i++) await sleep(5); + await sleep(10); + h.activity().onStartOfSpeech(vadEvent(VADEventType.START_OF_SPEECH)); + h.activity().onVADInferenceDone(vadEvent(VADEventType.INFERENCE_DONE, 600)); + await completed.waitForPlayout(); + await sleep(100); + expect(h.delivered()).toBe(FRAMES_PER_REPLY); + + // The user's transcript finalizes and cancelSpeechPause un-gates the output. + h.activity().onFinalTranscript(sttFinal('okay thanks'), false); + await sleep(150); + + const reply = await falseInterruptedReply(h, 'two'); + + expect(reply.pausedMidReply).toBe(true); + expect(reply.produced).toBe(FRAMES_PER_REPLY); + expect(reply.delivered).toBe(reply.produced); + } finally { + await h.close(); + } + }, 30000); +}); diff --git a/agents/src/voice/recorder_io/recorder_io.test.ts b/agents/src/voice/recorder_io/recorder_io.test.ts index c6f61aa42..df409ebf3 100644 --- a/agents/src/voice/recorder_io/recorder_io.test.ts +++ b/agents/src/voice/recorder_io/recorder_io.test.ts @@ -1032,6 +1032,10 @@ describe('RecorderAudioOutput in front of a real ParticipantAudioOutput', () => expect(await settleOrStall(output.waitForPlayout(), 2000)).not.toBe('did not settle'); output.resume(); + // The flush `forwardAudio` runs in its `finally` is what ends the interrupted segment. Without + // it the sink cannot tell a follow-on turn apart from the interrupted turn's TTS backlog, which + // keeps arriving after the resume that admits the next reply. + output.flush(); await output.captureFrame(makeFrame(100, 24000)); output.flush(); diff --git a/agents/src/voice/room_io/_output.test.ts b/agents/src/voice/room_io/_output.test.ts index 49a74c119..95d6e07ce 100644 --- a/agents/src/voice/room_io/_output.test.ts +++ b/agents/src/voice/room_io/_output.test.ts @@ -164,13 +164,20 @@ describe('ParticipantAudioOutput captureFrame segment accounting', () => { firstFrameEmitted: boolean; pushedDuration: number; _capturing: boolean; + interruptCount: number; + segmentInterruptCount: number; + segmentOpen: boolean; + gatedFrames: Set>; playbackSegmentsCount: number; playbackFinishedCount: number; playbackFinishedFuture: Future; onPlaybackStarted: (createdAt: number) => void; + logger: { error: () => void }; audioSource: { clearQueue: () => void; captureFrame: (frame: CaptureFrameArg) => Promise; + waitForPlayout: () => Promise; + queuedDuration: number; }; }; @@ -184,21 +191,39 @@ describe('ParticipantAudioOutput captureFrame segment accounting', () => { output.firstFrameEmitted = false; output.pushedDuration = 0; output._capturing = false; + output.interruptCount = 0; + output.segmentInterruptCount = 0; + output.segmentOpen = false; + output.gatedFrames = new Set(); output.playbackSegmentsCount = 0; output.playbackFinishedCount = 0; output.playbackFinishedFuture = new Future(); output.onPlaybackStarted = vi.fn(); - output.audioSource = { clearQueue: vi.fn(), captureFrame: vi.fn(async () => {}) }; + output.logger = { error: vi.fn() }; + output.audioSource = { + clearQueue: vi.fn(), + captureFrame: vi.fn(async () => {}), + // Playout never drains on its own, so a flush task stays pending like a real + // segment still on the wire. + waitForPlayout: () => new Promise(() => {}), + queuedDuration: 0, + }; return output; }; const frame = () => ({ samplesPerChannel: 480, sampleRate: 24000 }) as unknown as CaptureFrameArg; + const settledWithin = async (promise: Promise, ms: number) => + Promise.race([ + promise.then(() => 'settled' as const), + new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), ms)), + ]); + it('does not strand the segment counter when a frame is interrupted while paused', async () => { const output = makeOutput({ paused: true }); const capture = output.captureFrame(frame()); - output.interruptedFuture.resolve(); + output.clearBuffer(); await capture; expect(output.playbackSegmentsCount).toBe(0); @@ -211,6 +236,46 @@ describe('ParticipantAudioOutput captureFrame segment accounting', () => { expect(result).toBe('resolved'); }); + it('does not drop a new segment at the pause gate after the previous one was interrupted', async () => { + const output = makeOutput({ paused: false }); + + // Segment 1 plays, is flushed and then interrupted — the state a barge-in leaves behind. + await output.captureFrame(frame()); + output.flush(); + output.clearBuffer(); + await nextTick(); + + // Segment 2 is a fresh reply that gets paused mid-flight by a new overlap. Its frames + // belong to a speech the earlier interruption knows nothing about, so they must wait + // for the resume rather than be discarded. + output.pause(); + const capture = output.captureFrame(frame()); + expect(await settledWithin(capture, 50)).toBe('pending'); + + output.resume(); + await capture; + + expect(output.audioSource.captureFrame).toHaveBeenCalledTimes(2); + expect(output.playbackSegmentsCount).toBe(2); + }); + + it('releases a parked frame when a concurrent flush has replaced the interruption signal', async () => { + const output = makeOutput({ paused: false }); + + await output.captureFrame(frame()); + output.pause(); + const capture = output.captureFrame(frame()); + await nextTick(); + + // An overlapping segment's flush swaps interruptedFuture out from under the parked + // frame; the interruption that follows must still reach it. + output.flush(); + output.clearBuffer(); + + expect(await settledWithin(capture, 500)).toBe('settled'); + expect(output.audioSource.captureFrame).toHaveBeenCalledTimes(1); + }); + it('registers a segment on the normal non-paused path', async () => { const output = makeOutput({ paused: false }); diff --git a/agents/src/voice/room_io/_output.ts b/agents/src/voice/room_io/_output.ts index a78caa9fe..0c4d02c6d 100644 --- a/agents/src/voice/room_io/_output.ts +++ b/agents/src/voice/room_io/_output.ts @@ -384,6 +384,14 @@ export class ParticipantAudioOutput extends AudioOutput { private firstFrameEmitted: boolean = false; /** Gate held closed while the output is paused; frame forwarding awaits it. */ private playbackEnabledFuture: Future = new Future(); + /** Monotonic count of interruptions signalled through clearBuffer(). */ + private interruptCount: number = 0; + /** interruptCount as of the start of the segment currently being captured. */ + private segmentInterruptCount: number = 0; + /** Whether a frame has been captured since the last flush boundary. */ + private segmentOpen: boolean = false; + /** Wake-up futures for frames parked at the pause gate, all resolved by clearBuffer(). */ + private gatedFrames: Set> = new Set(); constructor(room: Room, options: AudioOutputOptions) { super(options.sampleRate, undefined, { pause: true }); @@ -423,17 +431,43 @@ export class ParticipantAudioOutput extends AudioOutput { } async captureFrame(frame: AudioFrame): Promise { + if (!this.segmentOpen) { + this.segmentOpen = true; + // An interruption raised before this segment began belongs to a speech that is already + // over. interruptedFuture stays resolved until the next flush, so without this snapshot + // every frame of the new speech bails at the gate below and the reply is lost. + this.segmentInterruptCount = this.interruptCount; + } + await this.startedFuture.await; if (!this.playbackEnabledFuture.done) { this.audioSource.clearQueue(); // Race against interruption so a cancel-while-paused can't deadlock an in-flight frame. - await Promise.race([this.playbackEnabledFuture.await, this.interruptedFuture.await]); - if (this.interruptedFuture.done) { - return; + // The wake-up is per frame rather than the shared interruptedFuture, which + // waitForPlayoutTask may replace while this frame is parked. + if (this.interruptCount === this.segmentInterruptCount) { + const gate = new Future(); + this.gatedFrames.add(gate); + try { + await Promise.race([this.playbackEnabledFuture.await, gate.await]); + } finally { + this.gatedFrames.delete(gate); + } } } + // Tested on every frame, not only the ones that parked at the gate. `cancelSpeechPause` + // un-gates the sink to admit the next reply as soon as the handle is interrupted, but the + // interrupted reply's `forwardAudio` loop only stops an event loop turn later, when its + // abort signal fires — and real TTS hands it seconds of audio ahead of realtime to drain in + // the meantime. Those frames find the gate already open, so without this they reach the wire + // while the next reply's transcript is streaming. `forwardAudio` always flushes in its + // `finally`, which is what opens a fresh segment for the next reply. + if (this.interruptCount > this.segmentInterruptCount) { + return; + } + // Count the playback segment only after the pause/interrupt gate above. super.captureFrame // bumps playbackSegmentsCount; if a frame interrupted-while-paused bailed at the gate after // that bump, the count would strand ahead of playbackFinishedCount and the next @@ -495,6 +529,7 @@ export class ParticipantAudioOutput extends AudioOutput { */ flush(): void { super.flush(); + this.segmentOpen = false; if (!this.pushedDuration) { return; @@ -522,10 +557,16 @@ export class ParticipantAudioOutput extends AudioOutput { } clearBuffer(): void { + this.interruptCount++; // Signal interruption even if no frame has been pushed yet, so a gated captureFrame can bail. if (!this.interruptedFuture.done) { this.interruptedFuture.resolve(); } + for (const gate of this.gatedFrames) { + if (!gate.done) { + gate.resolve(); + } + } } private async publishTrack(signal: AbortSignal) { diff --git a/agents/src/voice/room_io/_output_interrupted_segment.test.ts b/agents/src/voice/room_io/_output_interrupted_segment.test.ts new file mode 100644 index 000000000..c18ff40f0 --- /dev/null +++ b/agents/src/voice/room_io/_output_interrupted_segment.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame, type Room, TrackPublishOptions, TrackSource } from '@livekit/rtc-node'; +import { describe, expect, it } from 'vitest'; +import { ParticipantAudioOutput } from './_output.js'; + +const SAMPLE_RATE = 24000; +const FRAME_MS = 20; + +/** `marker` identifies which reply produced the frame once it reaches the wire. */ +function frame(marker: number): AudioFrame { + const samples = (SAMPLE_RATE * FRAME_MS) / 1000; + const data = new Int16Array(samples); + data[0] = marker; + return new AudioFrame(data, SAMPLE_RATE, 1, samples); +} + +/** The real output; only track publishing is skipped (no LiveKit server in a unit test). */ +class TestOutput extends ParticipantAudioOutput { + constructor() { + super({} as Room, { + sampleRate: SAMPLE_RATE, + numChannels: 1, + trackPublishOptions: new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }), + queueSizeMs: 100_000, + }); + (this as unknown as { startedFuture: { resolve: () => void } }).startedFuture.resolve(); + } + + override async start(): Promise {} +} + +/** Records the marker of every frame that reaches the wire. */ +function wireMarkers(out: ParticipantAudioOutput): number[] { + const source = ( + out as unknown as { audioSource: { captureFrame: (f: AudioFrame) => Promise } } + ).audioSource; + const original = source.captureFrame.bind(source); + const markers: number[] = []; + source.captureFrame = async (f: AudioFrame) => { + markers.push(f.data[0]!); + return original(f); + }; + return markers; +} + +describe('a committed barge-in with TTS audio still in flight', () => { + /** + * Real TTS delivers several seconds of audio ahead of realtime, so at the moment of a barge-in + * the interrupted reply's `forwardAudio` loop is still holding a backlog of already-synthesized + * frames. It stays alive for the whole of the reply task's + * `cancelAndWait(forwardTasks, REPLY_TASK_CANCEL_TIMEOUT)`, and only flushes in its `finally` + * after that — well after `cancelSpeechPause` has un-gated the sink for the next reply. + * + * `captureFrame` only consults `interruptCount` while the pause gate is closed, so every one of + * those backlog frames takes the open-gate path and reaches the wire while the next reply's + * transcript is already streaming. That is the audio/transcript desync. + */ + it('does not put the interrupted reply’s backlog on the wire', async () => { + const out = new TestOutput(); + const markers = wireMarkers(out); + + // Reply A is playing. + await out.captureFrame(frame(1)); + + // The user barges in: VAD pauses the sink, then the commit clears and un-gates it. + out.pause(); + out.clearBuffer(); + out.resume(); + + // Reply A's forwarding loop has not noticed the abort yet and drains its backlog. + await out.captureFrame(frame(1)); + await out.captureFrame(frame(1)); + + expect(markers.filter((m) => m === 1)).toHaveLength(1); + + await out.close(); + }); + + /** Control: once reply A's forwarding loop unwinds and flushes, reply B must be audible. */ + it('plays the next reply after the interrupted one flushes', async () => { + const out = new TestOutput(); + const markers = wireMarkers(out); + + await out.captureFrame(frame(1)); + + out.pause(); + out.clearBuffer(); + out.resume(); + + await out.captureFrame(frame(1)); + + // `forwardAudio` flushes in a `finally`, closing the interrupted segment. + out.flush(); + + await out.captureFrame(frame(2)); + await out.captureFrame(frame(2)); + + expect(markers.filter((m) => m === 2)).toHaveLength(2); + + await out.close(); + }); +}); diff --git a/plugins/krisp/src/_frame_identity.test.ts b/plugins/krisp/src/_frame_identity.test.ts new file mode 100644 index 000000000..3f01a615d --- /dev/null +++ b/plugins/krisp/src/_frame_identity.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame } from '@livekit/rtc-node'; +import { describe, expect, it } from 'vitest'; +import { adoptLocalAudioFrame } from './_frame_identity.js'; + +/** + * Stands in for the `AudioFrame` of a second copy of `@livekit/rtc-node`: identical shape, same + * constructor name, unrelated identity. That is exactly what the CJS build produces when the + * Cloud backend pulls it in through `createRequire`. + */ +class ForeignAudioFrame { + constructor( + readonly data: Int16Array, + readonly sampleRate: number, + readonly channels: number, + readonly samplesPerChannel: number, + private readonly _userdata: Record = {}, + ) {} + + get userdata(): Record { + return this._userdata; + } +} + +describe('adoptLocalAudioFrame', () => { + it('adopts a frame built by another copy of rtc-node', () => { + const samples = new Int16Array([1, -2, 3, -4]); + const foreign = new ForeignAudioFrame(samples, 16000, 1, 4, { source: 'krisp' }); + + // The premise of the bug: this lookalike fails the identity check every consumer relies on. + expect(foreign instanceof AudioFrame).toBe(false); + + const adopted = adoptLocalAudioFrame(foreign as unknown as AudioFrame); + + expect(adopted instanceof AudioFrame).toBe(true); + expect(adopted.sampleRate).toBe(16000); + expect(adopted.channels).toBe(1); + expect(adopted.samplesPerChannel).toBe(4); + expect(adopted.userdata).toEqual({ source: 'krisp' }); + // Adopting must not copy the audio: this runs on every frame of every session. + expect(adopted.data).toBe(samples); + }); + + it('returns a local frame untouched', () => { + const local = new AudioFrame(new Int16Array([5, 6]), 48000, 1, 2); + + expect(adoptLocalAudioFrame(local)).toBe(local); + }); +}); diff --git a/plugins/krisp/src/_frame_identity.ts b/plugins/krisp/src/_frame_identity.ts new file mode 100644 index 000000000..1cb267ff6 --- /dev/null +++ b/plugins/krisp/src/_frame_identity.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame } from '@livekit/rtc-node'; + +/** + * Return a frame that belongs to *this* copy of `@livekit/rtc-node`. + * + * The Cloud backend is reached through `createRequire`, which resolves the internal package's + * `require` condition and so loads the CJS build of `@livekit/rtc-node` next to our ESM one. + * Frames it hands back are instances of that copy's `AudioFrame`, so `instanceof AudioFrame` + * fails everywhere downstream: audio recognition treats them as unrecognised sentinels and + * drops them, and adaptive interruption — never fed any audio — rules every barge-in a + * backchannel. `AudioFrame` is a plain data holder, so adopting one shares its samples rather + * than copying them. + */ +export function adoptLocalAudioFrame(frame: AudioFrame): AudioFrame { + if (frame instanceof AudioFrame) { + return frame; + } + + // Statically unreachable — the declared type *is* `AudioFrame`. It is reachable at runtime + // precisely because that type came from a different copy of the module. + const foreign = frame as unknown as { + data: Int16Array; + sampleRate: number; + channels: number; + samplesPerChannel: number; + userdata?: Record; + }; + + return new AudioFrame( + foreign.data, + foreign.sampleRate, + foreign.channels, + foreign.samplesPerChannel, + foreign.userdata, + ); +} diff --git a/plugins/krisp/src/viva_filter.ts b/plugins/krisp/src/viva_filter.ts index 93ed6631c..1b20a3cd4 100644 --- a/plugins/krisp/src/viva_filter.ts +++ b/plugins/krisp/src/viva_filter.ts @@ -22,6 +22,7 @@ import type { } from '@livekit/rtc-node'; import { FrameProcessor } from '@livekit/rtc-node'; import { createRequire } from 'node:module'; +import { adoptLocalAudioFrame } from './_frame_identity.js'; import { KrispLicenseFrameProcessor } from './_krisp.js'; import { type AuthProvider, @@ -182,7 +183,7 @@ export class KrispVivaFilter extends FrameProcessor { } process(frame: AudioFrame): AudioFrame { - return this.inner.process(frame); + return adoptLocalAudioFrame(this.inner.process(frame)); } close(): void {