feat(d-id): add D-ID avatar plugin (#1)

.changeset/tangy-frogs-bathe.md

@@ -0,0 +1,7 @@
+---
+'@livekit/agents-plugin-did': patch
+---
+
+feat(d-id): add D-ID avatar plugin
+
+Dispatches a D-ID v4 (expressive) avatar worker into a LiveKit room via `POST /v2/agents/{agent_id}/sessions/join` and routes the agent's audio to it through `voice.DataStreamAudioOutput`. Audio sample rate is configurable (16k / 24k / 48k, default 24k) via `AudioConfig`. See `examples/src/did_avatar.ts` for usage.

examples/package.json

@@ -29,6 +29,7 @@
     "@livekit/agents-plugin-bey": "workspace:*",
     "@livekit/agents-plugin-cartesia": "workspace:*",
     "@livekit/agents-plugin-deepgram": "workspace:*",
+    "@livekit/agents-plugin-did": "workspace:*",
     "@livekit/agents-plugin-elevenlabs": "workspace:*",
     "@livekit/agents-plugin-fishaudio": "workspace:*",
     "@livekit/agents-plugin-google": "workspace:*",

examples/src/did_avatar.ts

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type JobContext,
+  ServerOptions,
+  cli,
+  defineAgent,
+  log,
+  metrics,
+  voice,
+} from '@livekit/agents';
+import * as did from '@livekit/agents-plugin-did';
+import * as openai from '@livekit/agents-plugin-openai';
+import { fileURLToPath } from 'node:url';
+
+export default defineAgent({
+  entry: async (ctx: JobContext) => {
+    const agent = new voice.Agent({
+      instructions: 'Talk to me!',
+    });
+
+    const logger = log();
+    const session = new voice.AgentSession({
+      llm: new openai.realtime.RealtimeModel({
+        voice: 'alloy',
+      }),
+    });
+
+    await ctx.connect();
+
+    await session.start({
+      agent,
+      room: ctx.room,
+    });
+
+    const agentId = process.env.DID_AGENT_ID;
+    if (!agentId) {
+      throw new Error('DID_AGENT_ID must be set');
+    }
+
+    const avatar = new did.AvatarSession({ agentId });
+    await avatar.start(session, ctx.room);
+
+    session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
+      metrics.logMetrics(ev.metrics);
+    });
+
+    ctx.addShutdownCallback(async () => {
+      logger.info(
+        {
+          usage: session.usage,
+        },
+        'Session usage summary',
+      );
+    });
+
+    session.generateReply({
+      instructions: 'Say hello to the user.',
+    });
+  },
+});
+
+cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/did/README.md

@@ -0,0 +1,36 @@
+# D-ID plugin for LiveKit Agents
+
+Support for the [D-ID](https://d-id.com/) virtual avatar.
+
+See the [D-ID integration docs](https://docs.livekit.io/agents/models/avatar/plugins/did/) for more information.
+
+## Installation
+
+```bash
+npm install @livekit/agents-plugin-did
+```
+
+## Pre-requisites
+
+You'll need an API key from D-ID. It can be set as an environment variable: `DID_API_KEY`
+
+## Supported avatars
+
+This plugin only supports **v4 avatars** (type: `expressive`). Earlier avatar versions are not compatible. See the [D-ID Create Agent API](https://docs.d-id.com/reference/createagent) for details on creating a compatible agent.
+
+Example — creating an expressive agent via the D-ID API:
+
+```bash
+curl -X POST https://api.d-id.com/agents \
+  -H "Authorization: Basic <YOUR_API_KEY>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "presenter": {
+      "type": "expressive",
+      "presenter_id": "public_mia_elegant@avt_TJ0Tq5"
+    },
+    "preview_name": "My Expressive Agent"
+  }'
+```
+
+Use the agent ID from the response as the `agentId` parameter in the plugin.

plugins/did/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/did/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@livekit/agents-plugin-did",
+  "version": "1.4.4",
+  "description": "D-ID avatar plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"pnpm build:types\"",
+    "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "catalog:",
+    "@microsoft/api-extractor": "^7.35.0",
+    "pino": "^8.19.0",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "livekit-server-sdk": "^2.13.3"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "catalog:"
+  }
+}

plugins/did/src/api.ts

@@ -0,0 +1,148 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type APIConnectOptions,
+  APIConnectionError,
+  APIStatusError,
+  DEFAULT_API_CONNECT_OPTIONS,
+  intervalForRetry,
+} from '@livekit/agents';
+import { log } from './log.js';
+
+/** @public */
+export const DEFAULT_API_URL = 'https://api.d-id.com';
+
+/**
+ * Exception thrown when the D-ID plugin or D-ID service errors.
+ *
+ * @public
+ */
+export class DIDException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'DIDException';
+  }
+}
+
+/** @public */
+export interface JoinSessionTransport {
+  /** Transport provider. Always `livekit` for this plugin. */
+  provider: 'livekit';
+  /** LiveKit server URL the D-ID worker should connect to. */
+  server_url: string;
+  /** LiveKit JWT for the D-ID worker. */
+  token: string;
+  /** LiveKit room name to join. */
+  room_name: string;
+}
+
+/** @public */
+export interface JoinSessionAudioConfig {
+  /** Sample rate in Hz. Supported values: 16000, 24000, 48000. */
+  sample_rate: number;
+}
+
+/** @public */
+export interface JoinSessionOptions {
+  /** D-ID agent id. */
+  agentId: string;
+  /** Transport configuration passed to the D-ID join endpoint. */
+  transport: JoinSessionTransport;
+  /** Audio configuration passed to the D-ID join endpoint. */
+  audioConfig: JoinSessionAudioConfig;
+}
+
+/** @public */
+export interface DIDAPIOptions {
+  /** D-ID API key. Falls back to `DID_API_KEY`. */
+  apiKey?: string;
+  /** Override the D-ID API base URL. */
+  apiUrl?: string;
+  /** API retry/timeout options. */
+  connOptions?: APIConnectOptions;
+}
+
+/**
+ * Thin client for the D-ID HTTP API.
+ *
+ * @public
+ */
+export class DIDAPI {
+  private apiKey: string;
+  private apiUrl: string;
+  private connOptions: APIConnectOptions;
+
+  #logger = log();
+
+  constructor(options: DIDAPIOptions = {}) {
+    const apiKey = options.apiKey ?? process.env.DID_API_KEY ?? '';
+    if (!apiKey) {
+      throw new DIDException('DID_API_KEY must be set');
+    }
+
+    this.apiKey = apiKey;
+    this.apiUrl = options.apiUrl || DEFAULT_API_URL;
+    this.connOptions = options.connOptions || DEFAULT_API_CONNECT_OPTIONS;
+  }
+
+  async joinSession(options: JoinSessionOptions): Promise<string> {
+    const payload: Record<string, unknown> = {
+      transport: options.transport,
+      audio_config: options.audioConfig,
+    };
+
+    const responseData = (await this.post(
+      `v2/agents/${options.agentId}/sessions/join`,
+      payload,
+    )) as { id: string };
+    return responseData.id;
+  }
+
+  private async post(endpoint: string, payload: Record<string, unknown>): Promise<unknown> {
+    const url = `${this.apiUrl}/${endpoint}`;
+
+    for (let i = 0; i <= this.connOptions.maxRetry; i++) {
+      try {
+        const response = await fetch(url, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Basic ${this.apiKey}`,
+          },
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(this.connOptions.timeoutMs),
+        });
+
+        if (!response.ok) {
+          const text = await response.text();
+          throw new APIStatusError({
+            message: 'Server returned an error',
+            options: { statusCode: response.status, body: { error: text } },
+          });
+        }
+
+        return await response.json();
+      } catch (e) {
+        if (e instanceof APIStatusError && !e.retryable) {
+          throw e;
+        }
+        if (e instanceof APIConnectionError) {
+          this.#logger.warn({ error: String(e) }, 'failed to call d-id api');
+        } else {
+          this.#logger.error({ error: e }, 'failed to call d-id api');
+        }
+
+        if (i < this.connOptions.maxRetry) {
+          await new Promise((resolve) =>
+            setTimeout(resolve, intervalForRetry(this.connOptions, i)),
+          );
+        }
+      }
+    }
+
+    throw new APIConnectionError({
+      message: 'Failed to call D-ID API after all retries',
+    });
+  }
+}

plugins/did/src/avatar.test.ts

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { voice } from '@livekit/agents';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { AvatarSession } from './avatar.js';
+
+describe('DID AvatarSession', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('calls base AvatarSession.start first', async () => {
+    const sentinel = new Error('super-start-called');
+    const superStartSpy = vi
+      .spyOn(voice.AvatarSession.prototype, 'start')
+      .mockRejectedValue(sentinel);
+
+    const avatar = new AvatarSession({
+      agentId: 'test-agent-id',
+      apiKey: 'test-api-key',
+    });
+
+    await expect(
+      avatar.start({ _started: false, output: { audio: null } } as any, {} as any),
+    ).rejects.toThrow('super-start-called');
+    expect(superStartSpy).toHaveBeenCalledTimes(1);
+  });
+});