diff --git a/.changeset/fuzzy-birds-barge.md b/.changeset/fuzzy-birds-barge.md new file mode 100644 index 000000000..92cafd170 --- /dev/null +++ b/.changeset/fuzzy-birds-barge.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Support adaptive interruption gating for realtime models without server-side turn detection. diff --git a/agents/src/inference/interruption/interruption_failover.test.ts b/agents/src/inference/interruption/interruption_failover.test.ts index d70b6538a..2def2af47 100644 --- a/agents/src/inference/interruption/interruption_failover.test.ts +++ b/agents/src/inference/interruption/interruption_failover.test.ts @@ -341,6 +341,7 @@ describe('interruption updateOptions reconnect', () => { function createHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/inference/interruption/interruption_stream.ts b/agents/src/inference/interruption/interruption_stream.ts index 486a95eb3..efb02819f 100644 --- a/agents/src/inference/interruption/interruption_stream.ts +++ b/agents/src/inference/interruption/interruption_stream.ts @@ -54,8 +54,8 @@ export class InterruptionStreamSentinel { return { type: 'overlap-speech-started', speechDuration, startedAt, userSpeakingSpan }; } - static overlapSpeechEnded(endedAt: number): OverlapSpeechEnded { - return { type: 'overlap-speech-ended', endedAt }; + static overlapSpeechEnded(endedAt: number, agentEnded = false): OverlapSpeechEnded { + return { type: 'overlap-speech-ended', endedAt, agentEnded }; } static flush(): Flush { @@ -287,6 +287,7 @@ export class InterruptionStreamBase { type: 'overlapping_speech', detectedAt: chunk.endedAt, isInterruption: false, + agentEnded: chunk.agentEnded, overlapStartedAt: this.overlapSpeechStartedAt, speechInput: e.speechInput, probabilities: e.probabilities, diff --git a/agents/src/inference/interruption/types.ts b/agents/src/inference/interruption/types.ts index 8306684a7..d062eb739 100644 --- a/agents/src/inference/interruption/types.ts +++ b/agents/src/inference/interruption/types.ts @@ -7,6 +7,12 @@ export interface OverlappingSpeechEvent { type: 'overlapping_speech'; detectedAt: number; isInterruption: boolean; + /** + * True when the overlap ended because the agent finished speaking rather than the user. + * The user may still be talking, so `isInterruption` (always false here) is inconclusive + * and must not be treated as a confirmed backchannel verdict. + */ + agentEnded?: boolean; totalDurationInS: number; predictionDurationInS: number; detectionDelayInS: number; @@ -66,6 +72,8 @@ export interface OverlapSpeechEnded { type: 'overlap-speech-ended'; /** Absolute timestamp (ms) when overlap speech ended, used as the non-interruption event timestamp. */ endedAt: number; + /** Whether the overlap ended because agent speech ended, not because user speech ended. */ + agentEnded?: boolean; } export interface Flush { diff --git a/agents/src/inference/interruption/ws_transport.ts b/agents/src/inference/interruption/ws_transport.ts index f0bd91b9f..a1b252660 100644 --- a/agents/src/inference/interruption/ws_transport.ts +++ b/agents/src/inference/interruption/ws_transport.ts @@ -309,6 +309,7 @@ export function createWsTransport( type: 'overlapping_speech', detectedAt: Date.now(), isInterruption: true, + agentEnded: false, totalDurationInS: entry.totalDurationInS, predictionDurationInS: entry.predictionDurationInS, overlapStartedAt: overlapSpeechStartedAt, diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 837ddb5e4..52d9727bb 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -296,6 +296,7 @@ export class AgentActivity implements RecognitionHooks { private readonly closeAbort = new AbortController(); private interruptionDetector?: AdaptiveInterruptionDetector; private isInterruptionDetectionEnabled: boolean; + private interruptionDetected = false; private isInterruptionByAudioActivityEnabled: boolean; private isDefaultInterruptionByAudioActivityEnabled: boolean; @@ -331,6 +332,7 @@ export class AgentActivity implements RecognitionHooks { this.onError(ev); private readonly onInterruptionOverlappingSpeech = (ev: OverlappingSpeechEvent): void => { + this.interruptionDetected = ev.isInterruption; this.agentSession.emit(AgentSessionEventTypes.OverlappingSpeech, ev); }; @@ -1381,6 +1383,7 @@ export class AgentActivity implements RecognitionHooks { this.agentSession._userSpeakingSpan, ); } + this.interruptionDetected = false; if (this.falseInterruptionTimer) { // cancel the timer when user starts speaking but leave the paused state unchanged @@ -1445,6 +1448,17 @@ export class AgentActivity implements RecognitionHooks { } } + onBackchannelConfirmed(): void { + if ( + this.isInterruptionDetectionEnabled && + this.realtimeSession !== undefined && + this.turnDetection !== 'manual' && + this.turnDetection !== 'realtime_llm' + ) { + this.realtimeSession.clearAudio(); + } + } + private interruptByAudioActivity(options?: { ignoreUserTranscriptUntil?: number }): void { if (!this.isInterruptionByAudioActivityEnabled) { return; @@ -1880,6 +1894,22 @@ export class AgentActivity implements RecognitionHooks { } } + if ( + !this.stt && + this.turnDetection !== 'manual' && + this.llm instanceof RealtimeModel && + !this.llm.capabilities.turnDetection && + this.isInterruptionDetectionEnabled && + (info.backchannelOverAgent || + (!this.interruptionDetected && + this._currentSpeech !== undefined && + !this._currentSpeech.interrupted)) + ) { + this.cancelPreemptiveGeneration(); + this.realtimeSession?.clearAudio(); + return false; + } + const oldTask = this._userTurnCompletedTask; this._userTurnCompletedTask = this.createSpeechTask({ taskFn: () => this.userTurnCompleted(info, oldTask), @@ -4371,16 +4401,25 @@ export class AgentActivity implements RecognitionHooks { private resolveInterruptionDetector(): AdaptiveInterruptionDetector | undefined { const agentInterruptionDetection = this.agent.turnHandling?.interruption?.mode; const sessionInterruptionDetection = this.agentSession.interruptionDetection; - if ( - !( + + let canGatekeep: boolean; + if (this.llm instanceof RealtimeModel) { + // Realtime commits turns manually; barge-in withholds the commit, so no STT is needed. + canGatekeep = !this.llm.capabilities.turnDetection; + } else { + // The STT pipeline gatekeeps by holding and flushing transcripts. + canGatekeep = !!( this.stt && this.stt.capabilities.alignedTranscript && - this.stt.capabilities.streaming && - this.vad !== undefined && - this.turnDetection !== 'manual' && - this.turnDetection !== 'realtime_llm' && - !(this.llm instanceof RealtimeModel) - ) + this.stt.capabilities.streaming + ); + } + + if ( + !canGatekeep || + this.vad === undefined || + this.turnDetection === 'manual' || + this.turnDetection === 'realtime_llm' ) { if ( agentInterruptionDetection === 'adaptive' || diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index dc25a117b..d11ed4680 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -88,6 +88,8 @@ export interface EndOfTurnInfo { * caller drives its own `generateReply` (e.g. leaving a voicemail). */ skipReply?: boolean; + /** The turn's speech overlapped agent speech and was classified a backchannel. */ + backchannelOverAgent?: boolean; } type EndOfTurnMetrics = { @@ -139,6 +141,7 @@ export interface PreemptiveGenerationInfo { export interface RecognitionHooks { onInterruption: (ev: OverlappingSpeechEvent) => void; + onBackchannelConfirmed: () => void; onStartOfSpeech: (ev: VADEvent) => void; onVADInferenceDone: (ev: VADEvent) => void; onEndOfSpeech: (ev: VADEvent) => void; @@ -373,6 +376,8 @@ export class AudioRecognition { private isAgentSpeaking: boolean; private agentSpeechStartedAt?: number; private interruptionDetected?: boolean; + private overlapInCurrentTurn = false; + private turnBackchannelOverAgent = false; private interruptionStreamChannel?: StreamChannel; private closed = false; @@ -772,7 +777,7 @@ export class AudioRecognition { // so it does not emit a synthetic `isInterruption: false` event following a real // interruption. if (priorIgnoreUserTranscriptUntil === undefined) { - this.onEndOfOverlapSpeech(Date.now()); + this.onEndOfOverlapSpeech(Date.now(), undefined, true); } await this.flushHeldTranscripts(endCooldown); } @@ -784,6 +789,8 @@ export class AudioRecognition { if (!this.endpointing.overlapping) { this.endpointing.onStartOfSpeech(startedAt, true); } + this.turnBackchannelOverAgent = false; + this.overlapInCurrentTurn = true; this.trySendInterruptionSentinel( InterruptionStreamSentinel.overlapSpeechStarted( speechDuration, @@ -795,7 +802,7 @@ export class AudioRecognition { } /** End interruption inference when overlap speech ends. */ - async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span) { + async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span, agentEnded = false) { if (!this.isInterruptionEnabled) { return; } @@ -803,7 +810,9 @@ export class AudioRecognition { userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, 'false'); } - return this.trySendInterruptionSentinel(InterruptionStreamSentinel.overlapSpeechEnded(endedAt)); + return this.trySendInterruptionSentinel( + InterruptionStreamSentinel.overlapSpeechEnded(endedAt, agentEnded), + ); } /** @@ -1222,6 +1231,8 @@ export class AudioRecognition { const ctx = this.userTurnContext(span); this.endpointing.onStartOfSpeech(speechStartTime, this.isAgentSpeaking); this.interruptionDetected = undefined; + this.turnBackchannelOverAgent = false; + this.overlapInCurrentTurn = this.isAgentSpeaking; otelContext.with(ctx, () => { this.hooks.onStartOfSpeech({ type: VADEventType.START_OF_SPEECH, @@ -1317,6 +1328,13 @@ export class AudioRecognition { this.interruptionDetected = ev.isInterruption; + if (this.overlapInCurrentTurn && !ev.agentEnded) { + this.turnBackchannelOverAgent = !ev.isInterruption; + if (!ev.isInterruption && !this.speaking) { + this.hooks.onBackchannelConfirmed(); + } + } + if (ev.isInterruption) { this.hooks.onInterruption(ev); } @@ -1639,6 +1657,7 @@ export class AudioRecognition { endOfUtteranceDelay: metrics.endOfUtteranceDelay, startedSpeakingAt: metrics.startedSpeakingAt, stoppedSpeakingAt: metrics.stoppedSpeakingAt, + backchannelOverAgent: this.turnBackchannelOverAgent, }); if (committed) { @@ -1670,6 +1689,8 @@ export class AudioRecognition { } } + this.turnBackchannelOverAgent = false; + this.overlapInCurrentTurn = false; this.userTurnCommitted = false; }; @@ -1833,6 +1854,8 @@ export class AudioRecognition { const ctx = this.userTurnContext(span); this.endpointing.onStartOfSpeech(startTime, this.isAgentSpeaking); this.interruptionDetected = undefined; + this.turnBackchannelOverAgent = false; + this.overlapInCurrentTurn = this.isAgentSpeaking; otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev)); } this.speaking = true; diff --git a/agents/src/voice/audio_recognition_backchannel.test.ts b/agents/src/voice/audio_recognition_backchannel.test.ts index 009bd82f0..62a047aaf 100644 --- a/agents/src/voice/audio_recognition_backchannel.test.ts +++ b/agents/src/voice/audio_recognition_backchannel.test.ts @@ -14,6 +14,7 @@ import { function createHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), @@ -48,6 +49,7 @@ function overlapSpeechEvent(isInterruption: boolean): OverlappingSpeechEvent { type: 'overlapping_speech', detectedAt: Date.now(), isInterruption, + agentEnded: false, totalDurationInS: 0, predictionDurationInS: 0, detectionDelayInS: 0, diff --git a/agents/src/voice/audio_recognition_duplicate_commit.test.ts b/agents/src/voice/audio_recognition_duplicate_commit.test.ts index 25cf12a05..3f11b52f0 100644 --- a/agents/src/voice/audio_recognition_duplicate_commit.test.ts +++ b/agents/src/voice/audio_recognition_duplicate_commit.test.ts @@ -67,6 +67,7 @@ describe('AudioRecognition duplicate EOU commit', () => { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_endpointing.test.ts b/agents/src/voice/audio_recognition_endpointing.test.ts index ec0d09533..0cd06dbbe 100644 --- a/agents/src/voice/audio_recognition_endpointing.test.ts +++ b/agents/src/voice/audio_recognition_endpointing.test.ts @@ -10,6 +10,7 @@ import { BaseEndpointing } from './turn_config/endpointing.js'; function createHooks(): RecognitionHooks { return { onInterruption: () => {}, + onBackchannelConfirmed: () => {}, onStartOfSpeech: () => {}, onVADInferenceDone: () => {}, onEndOfSpeech: () => {}, diff --git a/agents/src/voice/audio_recognition_eou.test.ts b/agents/src/voice/audio_recognition_eou.test.ts index 1431ff64d..bca659142 100644 --- a/agents/src/voice/audio_recognition_eou.test.ts +++ b/agents/src/voice/audio_recognition_eou.test.ts @@ -51,6 +51,7 @@ class SilentVAD extends VAD { function createHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_handoff.test.ts b/agents/src/voice/audio_recognition_handoff.test.ts index a6b395e47..8ebb51235 100644 --- a/agents/src/voice/audio_recognition_handoff.test.ts +++ b/agents/src/voice/audio_recognition_handoff.test.ts @@ -12,6 +12,7 @@ import type { STTNode } from './io.js'; function createHooks() { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_interruption.test.ts b/agents/src/voice/audio_recognition_interruption.test.ts index 03e60d5af..fc5630bfa 100644 --- a/agents/src/voice/audio_recognition_interruption.test.ts +++ b/agents/src/voice/audio_recognition_interruption.test.ts @@ -10,6 +10,7 @@ import { AudioRecognition, type RecognitionHooks } from './audio_recognition.js' function createHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_push_audio.test.ts b/agents/src/voice/audio_recognition_push_audio.test.ts index ddfa18a07..1d4e3586d 100644 --- a/agents/src/voice/audio_recognition_push_audio.test.ts +++ b/agents/src/voice/audio_recognition_push_audio.test.ts @@ -13,6 +13,7 @@ import { createSilenceFrame, createSilenceFrameLike } from './utils.js'; function createHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_span.test.ts b/agents/src/voice/audio_recognition_span.test.ts index 6a79698cd..e3e0f9a8c 100644 --- a/agents/src/voice/audio_recognition_span.test.ts +++ b/agents/src/voice/audio_recognition_span.test.ts @@ -104,6 +104,7 @@ describe('AudioRecognition user_turn span', () => { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), @@ -199,6 +200,7 @@ describe('AudioRecognition user_turn span', () => { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), @@ -272,6 +274,7 @@ describe('AudioRecognition user_turn span', () => { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), @@ -373,6 +376,7 @@ describe('AudioRecognition user_turn span', () => { const hooks: RecognitionHooks = { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_turn_detection.test.ts b/agents/src/voice/audio_recognition_turn_detection.test.ts index 9bf654317..c92e86938 100644 --- a/agents/src/voice/audio_recognition_turn_detection.test.ts +++ b/agents/src/voice/audio_recognition_turn_detection.test.ts @@ -75,6 +75,7 @@ interface RecognitionInternals { function makeHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/audio_recognition_vad_reset.test.ts b/agents/src/voice/audio_recognition_vad_reset.test.ts index 8ec404fb0..34085ec6c 100644 --- a/agents/src/voice/audio_recognition_vad_reset.test.ts +++ b/agents/src/voice/audio_recognition_vad_reset.test.ts @@ -27,6 +27,7 @@ interface RecognitionInternals { function makeHooks(): RecognitionHooks { return { onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), onStartOfSpeech: vi.fn(), onVADInferenceDone: vi.fn(), onEndOfSpeech: vi.fn(), diff --git a/agents/src/voice/realtime_adaptive_interruption.test.ts b/agents/src/voice/realtime_adaptive_interruption.test.ts new file mode 100644 index 000000000..533778ec0 --- /dev/null +++ b/agents/src/voice/realtime_adaptive_interruption.test.ts @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it, vi } from 'vitest'; +import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; +import { type RealtimeCapabilities, RealtimeModel, type RealtimeSession } from '../llm/realtime.js'; +import type { VADStream } from '../vad.js'; +import { VAD as BaseVAD } from '../vad.js'; +import { Agent } from './agent.js'; +import { AgentActivity } from './agent_activity.js'; +import { AgentSession } from './agent_session.js'; +import { AudioRecognition, type EndOfTurnInfo } from './audio_recognition.js'; +import { SpeechHandle } from './speech_handle.js'; +import { FakeLLM } from './testing/fake_llm.js'; + +class FakeVAD extends BaseVAD { + label = 'FakeVAD'; + + constructor() { + super({ updateInterval: 32 }); + } + + stream(): VADStream { + throw new Error('not used in this test'); + } +} + +class FakeRealtimeModel extends RealtimeModel { + get model() { + return 'fake-realtime'; + } + + session(): RealtimeSession { + throw new Error('not used in this test'); + } + + async close() {} +} + +function fakeCapabilities(overrides: Partial = {}): RealtimeCapabilities { + return { + messageTruncation: false, + turnDetection: false, + userTranscription: false, + autoToolReplyGeneration: false, + audioOutput: true, + manualFunctionCalls: false, + midSessionChatCtxUpdate: false, + midSessionInstructionsUpdate: false, + midSessionToolsUpdate: false, + ...overrides, + }; +} + +function realtimeBargeInSession(): AgentSession { + return new AgentSession({ + llm: new FakeRealtimeModel(fakeCapabilities({ turnDetection: false })), + vad: new FakeVAD(), + turnHandling: { + turnDetection: 'vad', + interruption: { mode: 'adaptive' }, + }, + }); +} + +function makeActivity(session: AgentSession): AgentActivity { + return new AgentActivity(new Agent({ instructions: 'test' }), session); +} + +function endOfTurnInfo(options: { backchannelOverAgent?: boolean } = {}): EndOfTurnInfo { + return { + newTranscript: '', + transcriptConfidence: 0, + transcriptionDelay: undefined, + endOfUtteranceDelay: undefined, + startedSpeakingAt: undefined, + stoppedSpeakingAt: undefined, + backchannelOverAgent: options.backchannelOverAgent ?? false, + }; +} + +type ActivityInternals = { + isInterruptionDetectionEnabled: boolean; + interruptionDetector?: unknown; + _schedulingPaused: boolean; + _currentSpeech?: SpeechHandle; + interruptionDetected: boolean; + realtimeSession?: { clearAudio: ReturnType }; + onEndOfTurn: (info: EndOfTurnInfo) => Promise; + onBackchannelConfirmed: () => void; +}; + +function setActivityProp(activity: object, key: string, value: T): void { + Object.defineProperty(activity, key, { configurable: true, value, writable: true }); +} + +describe('realtime adaptive interruption', () => { + it('enables adaptive interruption for realtime without STT', () => { + vi.stubEnv('LIVEKIT_API_KEY', 'k'); + vi.stubEnv('LIVEKIT_API_SECRET', 's'); + + const activity = makeActivity(realtimeBargeInSession()) as unknown as ActivityInternals; + + expect(activity.isInterruptionDetectionEnabled).toBe(true); + expect(activity.interruptionDetector).toBeDefined(); + }); + + it('still requires STT for non-realtime models', () => { + vi.stubEnv('LIVEKIT_API_KEY', 'k'); + vi.stubEnv('LIVEKIT_API_SECRET', 's'); + + const session = new AgentSession({ + llm: new FakeLLM([]), + vad: new FakeVAD(), + turnHandling: { + turnDetection: 'vad', + interruption: { mode: 'adaptive' }, + }, + }); + const activity = makeActivity(session) as unknown as ActivityInternals; + + expect(activity.isInterruptionDetectionEnabled).toBe(false); + expect(activity.interruptionDetector).toBeUndefined(); + }); + + it('disables adaptive interruption for realtime with server turn detection', () => { + vi.stubEnv('LIVEKIT_API_KEY', 'k'); + vi.stubEnv('LIVEKIT_API_SECRET', 's'); + + const session = new AgentSession({ + llm: new FakeRealtimeModel(fakeCapabilities({ turnDetection: true })), + vad: new FakeVAD(), + turnHandling: { interruption: { mode: 'adaptive' } }, + }); + const activity = makeActivity(session) as unknown as ActivityInternals; + + expect(activity.isInterruptionDetectionEnabled).toBe(false); + expect(activity.interruptionDetector).toBeUndefined(); + }); + + it('does not commit backchannels while agent speech is live', async () => { + vi.stubEnv('LIVEKIT_API_KEY', 'k'); + vi.stubEnv('LIVEKIT_API_SECRET', 's'); + + const activity = makeActivity(realtimeBargeInSession()) as unknown as ActivityInternals; + activity._schedulingPaused = false; + activity._currentSpeech = SpeechHandle.create({ allowInterruptions: true }); + activity.interruptionDetected = false; + + expect(await activity.onEndOfTurn(endOfTurnInfo())).toBe(false); + }); + + it('drops confirmed backchannels after agent speech finishes', async () => { + vi.stubEnv('LIVEKIT_API_KEY', 'k'); + vi.stubEnv('LIVEKIT_API_SECRET', 's'); + + const activity = makeActivity(realtimeBargeInSession()) as unknown as ActivityInternals; + activity._schedulingPaused = false; + activity._currentSpeech = undefined; + activity.interruptionDetected = false; + + expect(await activity.onEndOfTurn(endOfTurnInfo({ backchannelOverAgent: true }))).toBe(false); + }); + + it('clears realtime audio on confirmed backchannel even when STT exists', () => { + const activity = Object.create(AgentActivity.prototype) as ActivityInternals; + const realtimeSession = { clearAudio: vi.fn() }; + Object.assign(activity, { + isInterruptionDetectionEnabled: true, + realtimeSession, + }); + setActivityProp(activity, 'turnDetection', 'vad'); + + activity.onBackchannelConfirmed(); + + expect(realtimeSession.clearAudio).toHaveBeenCalledOnce(); + }); + + it('does not clear realtime audio when barge-in is disabled', () => { + const activity = Object.create(AgentActivity.prototype) as ActivityInternals; + const realtimeSession = { clearAudio: vi.fn() }; + Object.assign(activity, { + isInterruptionDetectionEnabled: false, + realtimeSession, + }); + setActivityProp(activity, 'turnDetection', 'vad'); + + activity.onBackchannelConfirmed(); + + expect(realtimeSession.clearAudio).not.toHaveBeenCalled(); + }); +}); + +type RecognitionInternals = { + backchannelBoundaryTimer?: ReturnType; + overlapInCurrentTurn: boolean; + turnBackchannelOverAgent: boolean; + speaking: boolean; + hooks: { + onInterruption: ReturnType; + onBackchannelConfirmed: ReturnType; + }; + onOverlapSpeechEvent: (ev: OverlappingSpeechEvent) => void; +}; + +function recognitionForOverlap(options: { speaking?: boolean } = {}): RecognitionInternals { + const recognition = Object.create(AudioRecognition.prototype) as RecognitionInternals; + Object.assign(recognition, { + backchannelBoundaryTimer: undefined, + overlapInCurrentTurn: true, + turnBackchannelOverAgent: false, + speaking: options.speaking ?? false, + hooks: { + onInterruption: vi.fn(), + onBackchannelConfirmed: vi.fn(), + }, + }); + return recognition; +} + +function overlapEvent(options: { + isInterruption: boolean; + agentEnded: boolean; +}): OverlappingSpeechEvent { + return { + type: 'overlapping_speech', + detectedAt: Date.now(), + isInterruption: options.isInterruption, + agentEnded: options.agentEnded, + totalDurationInS: 0, + predictionDurationInS: 0, + detectionDelayInS: 0, + probability: options.isInterruption ? 1 : 0, + numRequests: 0, + }; +} + +describe('AudioRecognition realtime adaptive backchannel verdicts', () => { + it('latches user-ended overlap as a backchannel', () => { + const recognition = recognitionForOverlap(); + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: false, agentEnded: false })); + expect(recognition.turnBackchannelOverAgent).toBe(true); + }); + + it('clears audio for confirmed backchannel between segments', () => { + const recognition = recognitionForOverlap({ speaking: false }); + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: false, agentEnded: false })); + expect(recognition.hooks.onBackchannelConfirmed).toHaveBeenCalledOnce(); + }); + + it('defers audio clear for confirmed backchannel while user is speaking', () => { + const recognition = recognitionForOverlap({ speaking: true }); + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: false, agentEnded: false })); + expect(recognition.turnBackchannelOverAgent).toBe(true); + expect(recognition.hooks.onBackchannelConfirmed).not.toHaveBeenCalled(); + }); + + it('does not treat agent-ended overlap as a backchannel', () => { + const recognition = recognitionForOverlap(); + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: false, agentEnded: true })); + expect(recognition.turnBackchannelOverAgent).toBe(false); + expect(recognition.hooks.onBackchannelConfirmed).not.toHaveBeenCalled(); + }); + + it('preserves a prior backchannel when a later agent-ended overlap arrives', () => { + const recognition = recognitionForOverlap(); + recognition.turnBackchannelOverAgent = true; + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: false, agentEnded: true })); + expect(recognition.turnBackchannelOverAgent).toBe(true); + }); + + it('clears backchannel verdict on interruption', () => { + const recognition = recognitionForOverlap(); + recognition.turnBackchannelOverAgent = true; + recognition.onOverlapSpeechEvent(overlapEvent({ isInterruption: true, agentEnded: false })); + expect(recognition.turnBackchannelOverAgent).toBe(false); + expect(recognition.hooks.onInterruption).toHaveBeenCalledOnce(); + expect(recognition.hooks.onBackchannelConfirmed).not.toHaveBeenCalled(); + }); +});