feat(eot): add backchannel support

.changeset/eot-backchannel-opportunity.md

@@ -0,0 +1,12 @@
+---
+"@livekit/agents": patch
+---
+
+feat(eot): emit agent backchannel opportunity events (AGT-2520)
+
+The multimodal EOT model now returns a backchannel probability alongside the end-of-turn probability. The turn detector compares it to a server-provided threshold and, when it clears, surfaces an internal backchannel *opportunity* (a window where the agent could say a short "mm-hmm" while the user still holds the floor) to `AgentActivity`.
+
+- `inference.TurnDetector` gains a `backchannelThreshold` option (and `updateOptions({ backchannelThreshold })`); `ThresholdOptions.lookupBackchannel()` resolves server-provided defaults layered with user overrides, mirroring the existing EOT threshold resolution.
+- Backchannel thresholds are server-driven and cloud-only — disabled when the gateway sends none, after a cloud→local fallback (the mini model produces no backchannel probability), and for any non-positive threshold.
+- Internal only: `AgentActivity.onAgentBackchannelOpportunity` is a no-op with a TODO; the event is not surfaced as a public `AgentSession` event (absent from the `AgentEvent` union, `AgentSessionEventTypes`, and package exports), treated the same way as the internal EOT prediction plumbing.
+- Requires `@livekit/protocol` >= 1.46.8 (adds `EotPrediction.backchannelProbability` and `SessionCreated.defaultBackchannelThresholds` / `defaultBackchannelThreshold`).

agents/package.json

@@ -54,7 +54,7 @@
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@livekit/local-inference": "^0.2.5",
     "@livekit/mutex": "^1.1.1",
-    "@livekit/protocol": "^1.46.5",
+    "@livekit/protocol": "^1.46.8",
     "@livekit/throws-transformer": "0.1.8",
     "@livekit/typed-emitter": "^3.0.0",
     "@opentelemetry/api": "^1.9.0",

agents/src/inference/eot/base.ts

@@ -45,6 +45,9 @@ export interface TurnDetectionEvent {
   detectionDelay?: number;
   /** Server-side model inference time (ms). */
   inferenceDuration?: number;
+  /** How appropriate it is for the agent to backchannel at this pause.
+   * `undefined` when the detector does not produce one (e.g. the local mini model). */
+  backchannelProbability?: number;
 }
 
 /**
@@ -125,6 +128,12 @@ export abstract class BaseStreamingTurnDetector extends (EventEmitter as new ()
     return this._opts.thresholds.lookup(language);
   }
 
+  /** Threshold above which a pause is a backchannel opportunity, or `undefined`
+   * when backchannel is disabled (server sent none, or the local mini model). */
+  async backchannelThreshold(language: LanguageCode | undefined): Promise<number | undefined> {
+    return this._opts.thresholds.lookupBackchannel(language);
+  }
+
   async supportsLanguage(language: LanguageCode | undefined): Promise<boolean> {
     return this._opts.thresholds.supports(language);
   }
@@ -219,6 +228,10 @@ export class BaseStreamingTurnDetectorStream {
     return this._opts.thresholds.lookup(language);
   }
 
+  async backchannelThreshold(language: LanguageCode | undefined): Promise<number | undefined> {
+    return this._opts.thresholds.lookupBackchannel(language);
+  }
+
   async supportsLanguage(language: LanguageCode | undefined): Promise<boolean> {
     return this._opts.thresholds.supports(language);
   }
@@ -367,7 +380,11 @@ export class BaseStreamingTurnDetectorStream {
   _resolvePrediction(
     requestId: string,
     probability: number,
-    opts: { inferenceDuration?: number; detectionDelay?: number } = {},
+    opts: {
+      inferenceDuration?: number;
+      detectionDelay?: number;
+      backchannelProbability?: number;
+    } = {},
   ): void {
     // Drop predictions that land after teardown — an in-flight transport
     // predict can resolve after `aclose` closed the channels.
@@ -387,6 +404,7 @@ export class BaseStreamingTurnDetectorStream {
         lastSpeakingTimeMs: Date.now(),
         detectionDelay: opts.detectionDelay,
         inferenceDuration: opts.inferenceDuration,
+        backchannelProbability: opts.backchannelProbability,
       });
     }
   }

agents/src/inference/eot/detector.test.ts

@@ -41,6 +41,10 @@ import { LocalTransport } from './transports.js';
 const SERVER_THRESHOLDS: Record<string, number> = { en: 0.56, ja: 0.37, fr: 0.575 };
 const SERVER_DEFAULT_THRESHOLD = 0.5;
 
+// Backchannel defaults a gateway returns alongside the EOT defaults.
+const SERVER_BACKCHANNEL_THRESHOLDS: Record<string, number> = { en: 0.62, ja: 0.7 };
+const SERVER_BACKCHANNEL_DEFAULT = 0.6;
+
 async function waitFor(predicate: () => boolean, ticks = 50): Promise<void> {
   for (let i = 0; i < ticks; i++) {
     if (predicate()) return;
@@ -429,6 +433,107 @@ describe('ResolveThresholds', () => {
   });
 });
 
+describe('BackchannelThresholds', () => {
+  // Server-provided backchannel defaults, disabled on the mini model and after
+  // fallback (see ResolveBackchannelThresholds for override layering).
+  function cloud(): ThresholdOptions {
+    const opts = new ThresholdOptions('turn-detector-v1');
+    opts._updateDefaults(
+      { ...SERVER_THRESHOLDS },
+      SERVER_DEFAULT_THRESHOLD,
+      { ...SERVER_BACKCHANNEL_THRESHOLDS },
+      SERVER_BACKCHANNEL_DEFAULT,
+    );
+    return opts;
+  }
+
+  it('lookup per-language and default', () => {
+    const opts = cloud();
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(SERVER_BACKCHANNEL_THRESHOLDS.en!);
+    // absent language → catch-all backchannel default
+    expect(opts.lookupBackchannel('de')).toBeCloseTo(SERVER_BACKCHANNEL_DEFAULT);
+    // undefined language defaults to "en"
+    expect(opts.lookupBackchannel(undefined)).toBeCloseTo(SERVER_BACKCHANNEL_THRESHOLDS.en!);
+  });
+
+  it('disabled when server omits backchannel', () => {
+    const opts = new ThresholdOptions('turn-detector-v1');
+    opts._updateDefaults({ ...SERVER_THRESHOLDS }, SERVER_DEFAULT_THRESHOLD);
+    expect(opts.lookupBackchannel('en')).toBeUndefined();
+  });
+
+  it('disabled for the local mini model', () => {
+    const opts = new ThresholdOptions('turn-detector-v1-mini');
+    expect(opts.lookupBackchannel('en')).toBeUndefined();
+  });
+
+  it('non-positive threshold treated as disabled', () => {
+    const opts = new ThresholdOptions('turn-detector-v1');
+    opts._updateDefaults({ ...SERVER_THRESHOLDS }, SERVER_DEFAULT_THRESHOLD, { en: 0.0 }, 0.6);
+    // en explicitly 0 → disabled for en, but the positive default still applies elsewhere
+    expect(opts.lookupBackchannel('en')).toBeUndefined();
+    expect(opts.lookupBackchannel('de')).toBeCloseTo(0.6);
+  });
+
+  it('cleared on local fallback', () => {
+    const opts = cloud();
+    opts._toLocalFallback();
+    expect(opts.lookupBackchannel('en')).toBeUndefined();
+  });
+});
+
+describe('ResolveBackchannelThresholds', () => {
+  // User backchannel-threshold overrides layered against the server defaults,
+  // mirroring the EOT override resolution in ResolveThresholds.
+  function cloud(overrides?: number | Record<string, number>): ThresholdOptions {
+    const opts = new ThresholdOptions('turn-detector-v1', undefined, overrides);
+    opts._updateDefaults(
+      { ...SERVER_THRESHOLDS },
+      SERVER_DEFAULT_THRESHOLD,
+      { ...SERVER_BACKCHANNEL_THRESHOLDS },
+      SERVER_BACKCHANNEL_DEFAULT,
+    );
+    return opts;
+  }
+
+  it('no override adopts server backchannel', () => {
+    const opts = cloud();
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(SERVER_BACKCHANNEL_THRESHOLDS.en!);
+  });
+
+  it('scalar override applies to every language', () => {
+    const opts = cloud(0.8);
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(0.8);
+    expect(opts.lookupBackchannel('ja')).toBeCloseTo(0.8);
+  });
+
+  it('dict override layers on server map', () => {
+    const opts = cloud({ en: 0.5 });
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(0.5);
+    // unmapped languages keep the server values + server default
+    expect(opts.lookupBackchannel('ja')).toBeCloseTo(SERVER_BACKCHANNEL_THRESHOLDS.ja!);
+    expect(opts.lookupBackchannel('de')).toBeCloseTo(SERVER_BACKCHANNEL_DEFAULT);
+  });
+
+  it('dict keys normalized', () => {
+    const opts = cloud({ English: 0.5 });
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(0.5);
+  });
+
+  it('scalar override enables before server defaults', () => {
+    // an explicit scalar override resolves up front, even though the server
+    // backchannel defaults haven't arrived yet
+    const opts = new ThresholdOptions('turn-detector-v1', undefined, 0.8);
+    expect(opts.lookupBackchannel('en')).toBeCloseTo(0.8);
+  });
+
+  it('updateBackchannelOverrides re-resolves', () => {
+    const opts = cloud();
+    opts.updateBackchannelOverrides(0.45);
+    expect(opts.lookupBackchannel('ja')).toBeCloseTo(0.45);
+  });
+});
+
 describe('ServerDefaults', () => {
   it('cloud thresholds pending before session created', async () => {
     const transport = new ScriptedTransport({ runBehavior: 'idle' });
@@ -492,6 +597,21 @@ describe('OverrideWarning', () => {
     }
   });
 
+  it('warns on construction with backchannel override', () => {
+    const warnSpy = vi.spyOn(log(), 'warn');
+    try {
+      withEnv({ LIVEKIT_REMOTE_EOT_URL: undefined }, () => {
+        new TurnDetector({ backchannelThreshold: 0.7 });
+      });
+      const warned = warnSpy.mock.calls.some((c) =>
+        JSON.stringify(c).includes('non-default backchannel threshold'),
+      );
+      expect(warned).toBe(true);
+    } finally {
+      warnSpy.mockRestore();
+    }
+  });
+
   it('no warning without override', () => {
     const warnSpy = vi.spyOn(log(), 'warn');
     try {

agents/src/inference/eot/detector.ts

@@ -33,6 +33,12 @@ export interface TurnDetectorOptions {
    */
   version?: TurnDetectorVersion;
   unlikelyThreshold?: number | Record<string, number>;
+  /**
+   * Backchannel threshold(s): above this, a pause is a backchannel opportunity.
+   * Server-driven and cloud-only by default; this is an override seam. A scalar
+   * applies to every language; a map is layered over the server defaults.
+   */
+  backchannelThreshold?: number | Record<string, number>;
   baseUrl?: string;
   apiKey?: string;
   apiSecret?: string;
@@ -103,7 +109,11 @@ export class TurnDetector extends BaseStreamingTurnDetector {
 
     const detectorOpts: BaseStreamingTurnDetectorOptions = {
       sampleRate: opts.sampleRate ?? DEFAULT_SAMPLE_RATE,
-      thresholds: new ThresholdOptions(resolvedModel, opts.unlikelyThreshold),
+      thresholds: new ThresholdOptions(
+        resolvedModel,
+        opts.unlikelyThreshold,
+        opts.backchannelThreshold,
+      ),
     };
     super(detectorOpts);
     this._model = resolvedModel;
@@ -146,13 +156,31 @@ export class TurnDetector extends BaseStreamingTurnDetector {
           'defaults and overriding them may be suboptimal',
       );
     }
+    const bcOverrides = this._opts.thresholds.backchannelOverrides;
+    if (bcOverrides !== undefined) {
+      log().warn(
+        { backchannelThreshold: bcOverrides },
+        'a non-default backchannel threshold was provided; the server provides calibrated ' +
+          'defaults and overriding them may be suboptimal',
+      );
+    }
   }
 
   /** Replace the user threshold override at runtime. The shared
    * `ThresholdOptions` re-resolves against the current (server or shipped)
    * defaults, so an active stream picks it up immediately. */
-  updateOptions(opts: { unlikelyThreshold?: number | Record<string, number> } = {}): void {
-    this._opts.thresholds.updateOverrides(opts.unlikelyThreshold);
+  updateOptions(
+    opts: {
+      unlikelyThreshold?: number | Record<string, number>;
+      backchannelThreshold?: number | Record<string, number>;
+    } = {},
+  ): void {
+    if (opts.unlikelyThreshold !== undefined) {
+      this._opts.thresholds.updateOverrides(opts.unlikelyThreshold);
+    }
+    if (opts.backchannelThreshold !== undefined) {
+      this._opts.thresholds.updateBackchannelOverrides(opts.backchannelThreshold);
+    }
     this._warnThresholdOverride();
   }