From db43145f8e36b95151e96ee6ee3dc96435d1a6ed Mon Sep 17 00:00:00 2001 From: Toubat Date: Sun, 26 Jul 2026 00:51:43 -0700 Subject: [PATCH] fix(krisp): adopt processed frames into the local rtc-node binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cloud backend is reached through createRequire, which resolves the internal package's `require` condition and so loads the CJS build of @livekit/rtc-node next to the ESM one the framework uses. Frames it returned were instances of that copy's AudioFrame, so every `instanceof AudioFrame` downstream failed: audio recognition dropped them as unrecognised sentinels, and adaptive interruption — fed no audio at all — ruled every barge-in a backchannel. Enabling noise cancellation therefore made the agent impossible to interrupt. Frames are now adopted into the local binding before leaving the filter. AudioFrame is a plain data holder, so this shares the samples rather than copying them. Co-authored-by: Cursor --- .changeset/fix-krisp-frame-identity.md | 13 ++++++ plugins/krisp/src/_frame_identity.test.ts | 51 +++++++++++++++++++++++ plugins/krisp/src/_frame_identity.ts | 39 +++++++++++++++++ plugins/krisp/src/viva_filter.ts | 3 +- 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 .changeset/fix-krisp-frame-identity.md create mode 100644 plugins/krisp/src/_frame_identity.test.ts create mode 100644 plugins/krisp/src/_frame_identity.ts diff --git a/.changeset/fix-krisp-frame-identity.md b/.changeset/fix-krisp-frame-identity.md new file mode 100644 index 000000000..36babff8d --- /dev/null +++ b/.changeset/fix-krisp-frame-identity.md @@ -0,0 +1,13 @@ +--- +'@livekit/agents-plugin-krisp': patch +--- + +Fix Krisp-processed audio being invisible to the rest of the pipeline + +The LiveKit Cloud backend is reached through `createRequire`, which resolves the internal +package's `require` condition and so loads the CJS build of `@livekit/rtc-node` next to the +ESM one the framework uses. Frames returned by that backend were instances of the CJS copy's +`AudioFrame`, so every `instanceof AudioFrame` downstream failed. Adaptive interruption saw +zero audio and classified every barge-in as a backchannel, making it impossible to interrupt +an agent that had noise cancellation enabled. Frames are now adopted into the local binding +before leaving the filter, sharing their samples rather than copying them. diff --git a/plugins/krisp/src/_frame_identity.test.ts b/plugins/krisp/src/_frame_identity.test.ts new file mode 100644 index 000000000..3f01a615d --- /dev/null +++ b/plugins/krisp/src/_frame_identity.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame } from '@livekit/rtc-node'; +import { describe, expect, it } from 'vitest'; +import { adoptLocalAudioFrame } from './_frame_identity.js'; + +/** + * Stands in for the `AudioFrame` of a second copy of `@livekit/rtc-node`: identical shape, same + * constructor name, unrelated identity. That is exactly what the CJS build produces when the + * Cloud backend pulls it in through `createRequire`. + */ +class ForeignAudioFrame { + constructor( + readonly data: Int16Array, + readonly sampleRate: number, + readonly channels: number, + readonly samplesPerChannel: number, + private readonly _userdata: Record = {}, + ) {} + + get userdata(): Record { + return this._userdata; + } +} + +describe('adoptLocalAudioFrame', () => { + it('adopts a frame built by another copy of rtc-node', () => { + const samples = new Int16Array([1, -2, 3, -4]); + const foreign = new ForeignAudioFrame(samples, 16000, 1, 4, { source: 'krisp' }); + + // The premise of the bug: this lookalike fails the identity check every consumer relies on. + expect(foreign instanceof AudioFrame).toBe(false); + + const adopted = adoptLocalAudioFrame(foreign as unknown as AudioFrame); + + expect(adopted instanceof AudioFrame).toBe(true); + expect(adopted.sampleRate).toBe(16000); + expect(adopted.channels).toBe(1); + expect(adopted.samplesPerChannel).toBe(4); + expect(adopted.userdata).toEqual({ source: 'krisp' }); + // Adopting must not copy the audio: this runs on every frame of every session. + expect(adopted.data).toBe(samples); + }); + + it('returns a local frame untouched', () => { + const local = new AudioFrame(new Int16Array([5, 6]), 48000, 1, 2); + + expect(adoptLocalAudioFrame(local)).toBe(local); + }); +}); diff --git a/plugins/krisp/src/_frame_identity.ts b/plugins/krisp/src/_frame_identity.ts new file mode 100644 index 000000000..1cb267ff6 --- /dev/null +++ b/plugins/krisp/src/_frame_identity.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { AudioFrame } from '@livekit/rtc-node'; + +/** + * Return a frame that belongs to *this* copy of `@livekit/rtc-node`. + * + * The Cloud backend is reached through `createRequire`, which resolves the internal package's + * `require` condition and so loads the CJS build of `@livekit/rtc-node` next to our ESM one. + * Frames it hands back are instances of that copy's `AudioFrame`, so `instanceof AudioFrame` + * fails everywhere downstream: audio recognition treats them as unrecognised sentinels and + * drops them, and adaptive interruption — never fed any audio — rules every barge-in a + * backchannel. `AudioFrame` is a plain data holder, so adopting one shares its samples rather + * than copying them. + */ +export function adoptLocalAudioFrame(frame: AudioFrame): AudioFrame { + if (frame instanceof AudioFrame) { + return frame; + } + + // Statically unreachable — the declared type *is* `AudioFrame`. It is reachable at runtime + // precisely because that type came from a different copy of the module. + const foreign = frame as unknown as { + data: Int16Array; + sampleRate: number; + channels: number; + samplesPerChannel: number; + userdata?: Record; + }; + + return new AudioFrame( + foreign.data, + foreign.sampleRate, + foreign.channels, + foreign.samplesPerChannel, + foreign.userdata, + ); +} diff --git a/plugins/krisp/src/viva_filter.ts b/plugins/krisp/src/viva_filter.ts index 93ed6631c..1b20a3cd4 100644 --- a/plugins/krisp/src/viva_filter.ts +++ b/plugins/krisp/src/viva_filter.ts @@ -22,6 +22,7 @@ import type { } from '@livekit/rtc-node'; import { FrameProcessor } from '@livekit/rtc-node'; import { createRequire } from 'node:module'; +import { adoptLocalAudioFrame } from './_frame_identity.js'; import { KrispLicenseFrameProcessor } from './_krisp.js'; import { type AuthProvider, @@ -182,7 +183,7 @@ export class KrispVivaFilter extends FrameProcessor { } process(frame: AudioFrame): AudioFrame { - return this.inner.process(frame); + return adoptLocalAudioFrame(this.inner.process(frame)); } close(): void {