Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .changeset/fix-krisp-frame-identity.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
'@livekit/agents-plugin-krisp': patch
---

Fix Krisp-processed audio being invisible to the rest of the pipeline

The LiveKit Cloud backend is reached through `createRequire`, which resolves the internal
package's `require` condition and so loads the CJS build of `@livekit/rtc-node` next to the
ESM one the framework uses. Frames returned by that backend were instances of the CJS copy's
`AudioFrame`, so every `instanceof AudioFrame` downstream failed. Adaptive interruption saw
zero audio and classified every barge-in as a backchannel, making it impossible to interrupt
an agent that had noise cancellation enabled. Frames are now adopted into the local binding
before leaving the filter, sharing their samples rather than copying them.
51 changes: 51 additions & 0 deletions plugins/krisp/src/_frame_identity.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
//
// SPDX-License-Identifier: Apache-2.0
import { AudioFrame } from '@livekit/rtc-node';
import { describe, expect, it } from 'vitest';
import { adoptLocalAudioFrame } from './_frame_identity.js';

/**
* Stands in for the `AudioFrame` of a second copy of `@livekit/rtc-node`: identical shape, same
* constructor name, unrelated identity. That is exactly what the CJS build produces when the
* Cloud backend pulls it in through `createRequire`.
*/
class ForeignAudioFrame {
constructor(
readonly data: Int16Array,
readonly sampleRate: number,
readonly channels: number,
readonly samplesPerChannel: number,
private readonly _userdata: Record<string, unknown> = {},
) {}

get userdata(): Record<string, unknown> {
return this._userdata;
}
}

describe('adoptLocalAudioFrame', () => {
it('adopts a frame built by another copy of rtc-node', () => {
const samples = new Int16Array([1, -2, 3, -4]);
const foreign = new ForeignAudioFrame(samples, 16000, 1, 4, { source: 'krisp' });

// The premise of the bug: this lookalike fails the identity check every consumer relies on.
expect(foreign instanceof AudioFrame).toBe(false);

const adopted = adoptLocalAudioFrame(foreign as unknown as AudioFrame);

expect(adopted instanceof AudioFrame).toBe(true);
expect(adopted.sampleRate).toBe(16000);
expect(adopted.channels).toBe(1);
expect(adopted.samplesPerChannel).toBe(4);
expect(adopted.userdata).toEqual({ source: 'krisp' });
// Adopting must not copy the audio: this runs on every frame of every session.
expect(adopted.data).toBe(samples);
});

it('returns a local frame untouched', () => {
const local = new AudioFrame(new Int16Array([5, 6]), 48000, 1, 2);

expect(adoptLocalAudioFrame(local)).toBe(local);
});
});
39 changes: 39 additions & 0 deletions plugins/krisp/src/_frame_identity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
//
// SPDX-License-Identifier: Apache-2.0
import { AudioFrame } from '@livekit/rtc-node';

/**
* Return a frame that belongs to *this* copy of `@livekit/rtc-node`.
*
* The Cloud backend is reached through `createRequire`, which resolves the internal package's
* `require` condition and so loads the CJS build of `@livekit/rtc-node` next to our ESM one.
* Frames it hands back are instances of that copy's `AudioFrame`, so `instanceof AudioFrame`
* fails everywhere downstream: audio recognition treats them as unrecognised sentinels and
* drops them, and adaptive interruption — never fed any audio — rules every barge-in a
* backchannel. `AudioFrame` is a plain data holder, so adopting one shares its samples rather
* than copying them.
*/
export function adoptLocalAudioFrame(frame: AudioFrame): AudioFrame {
if (frame instanceof AudioFrame) {
return frame;
}

// Statically unreachable — the declared type *is* `AudioFrame`. It is reachable at runtime
// precisely because that type came from a different copy of the module.
const foreign = frame as unknown as {
data: Int16Array;
sampleRate: number;
channels: number;
samplesPerChannel: number;
userdata?: Record<string, unknown>;
};

return new AudioFrame(
foreign.data,
foreign.sampleRate,
foreign.channels,
foreign.samplesPerChannel,
foreign.userdata,
);
}
3 changes: 2 additions & 1 deletion plugins/krisp/src/viva_filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import type {
} from '@livekit/rtc-node';
import { FrameProcessor } from '@livekit/rtc-node';
import { createRequire } from 'node:module';
import { adoptLocalAudioFrame } from './_frame_identity.js';
import { KrispLicenseFrameProcessor } from './_krisp.js';
import {
type AuthProvider,
Expand Down Expand Up @@ -182,7 +183,7 @@ export class KrispVivaFilter extends FrameProcessor<AudioFrame> {
}

process(frame: AudioFrame): AudioFrame {
return this.inner.process(frame);
return adoptLocalAudioFrame(this.inner.process(frame));
}

close(): void {
Expand Down
Loading