diff --git a/.changeset/tangy-frogs-bathe.md b/.changeset/tangy-frogs-bathe.md new file mode 100644 index 000000000..cdf388231 --- /dev/null +++ b/.changeset/tangy-frogs-bathe.md @@ -0,0 +1,7 @@ +--- +'@livekit/agents-plugin-did': patch +--- + +feat(d-id): add D-ID avatar plugin + +Dispatches a D-ID v4 (expressive) avatar worker into a LiveKit room via `POST /v2/agents/{agent_id}/sessions/join` and routes the agent's audio to it through `voice.DataStreamAudioOutput`. Audio sample rate is configurable (16k / 24k / 48k, default 24k) via `AudioConfig`. See `examples/src/did_avatar.ts` for usage. diff --git a/examples/package.json b/examples/package.json index 851fad6fe..66c0a13b3 100644 --- a/examples/package.json +++ b/examples/package.json @@ -29,6 +29,7 @@ "@livekit/agents-plugin-bey": "workspace:*", "@livekit/agents-plugin-cartesia": "workspace:*", "@livekit/agents-plugin-deepgram": "workspace:*", + "@livekit/agents-plugin-did": "workspace:*", "@livekit/agents-plugin-elevenlabs": "workspace:*", "@livekit/agents-plugin-fishaudio": "workspace:*", "@livekit/agents-plugin-google": "workspace:*", diff --git a/examples/src/did_avatar.ts b/examples/src/did_avatar.ts new file mode 100644 index 000000000..65cdfa6dc --- /dev/null +++ b/examples/src/did_avatar.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + log, + metrics, + voice, +} from '@livekit/agents'; +import * as did from '@livekit/agents-plugin-did'; +import * as openai from '@livekit/agents-plugin-openai'; +import { fileURLToPath } from 'node:url'; + +export default defineAgent({ + entry: async (ctx: JobContext) => { + const agent = new voice.Agent({ + instructions: 'Talk to me!', + }); + + const logger = log(); + const session = new voice.AgentSession({ + llm: new openai.realtime.RealtimeModel({ + voice: 'alloy', + }), + }); + + await ctx.connect(); + + await session.start({ + agent, + room: ctx.room, + }); + + const agentId = process.env.DID_AGENT_ID; + if (!agentId) { + throw new Error('DID_AGENT_ID must be set'); + } + + const avatar = new did.AvatarSession({ agentId }); + await avatar.start(session, ctx.room); + + session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { + metrics.logMetrics(ev.metrics); + }); + + ctx.addShutdownCallback(async () => { + logger.info( + { + usage: session.usage, + }, + 'Session usage summary', + ); + }); + + session.generateReply({ + instructions: 'Say hello to the user.', + }); + }, +}); + +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/plugins/did/README.md b/plugins/did/README.md new file mode 100644 index 000000000..f9d96d1aa --- /dev/null +++ b/plugins/did/README.md @@ -0,0 +1,36 @@ +# D-ID plugin for LiveKit Agents + +Support for the [D-ID](https://d-id.com/) virtual avatar. + +See the [D-ID integration docs](https://docs.livekit.io/agents/models/avatar/plugins/did/) for more information. + +## Installation + +```bash +npm install @livekit/agents-plugin-did +``` + +## Pre-requisites + +You'll need an API key from D-ID. It can be set as an environment variable: `DID_API_KEY` + +## Supported avatars + +This plugin only supports **v4 avatars** (type: `expressive`). Earlier avatar versions are not compatible. See the [D-ID Create Agent API](https://docs.d-id.com/reference/createagent) for details on creating a compatible agent. + +Example — creating an expressive agent via the D-ID API: + +```bash +curl -X POST https://api.d-id.com/agents \ + -H "Authorization: Basic " \ + -H "Content-Type: application/json" \ + -d '{ + "presenter": { + "type": "expressive", + "presenter_id": "public_mia_elegant@avt_TJ0Tq5" + }, + "preview_name": "My Expressive Agent" + }' +``` + +Use the agent ID from the response as the `agentId` parameter in the plugin. diff --git a/plugins/did/api-extractor.json b/plugins/did/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/did/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/did/package.json b/plugins/did/package.json new file mode 100644 index 000000000..786af7a9b --- /dev/null +++ b/plugins/did/package.json @@ -0,0 +1,51 @@ +{ + "name": "@livekit/agents-plugin-did", + "version": "1.4.4", + "description": "D-ID avatar plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "pino": "^8.19.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "livekit-server-sdk": "^2.13.3" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/did/src/api.ts b/plugins/did/src/api.ts new file mode 100644 index 000000000..184b84f4f --- /dev/null +++ b/plugins/did/src/api.ts @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APIStatusError, + DEFAULT_API_CONNECT_OPTIONS, + intervalForRetry, +} from '@livekit/agents'; +import { log } from './log.js'; + +/** @public */ +export const DEFAULT_API_URL = 'https://api.d-id.com'; + +/** + * Exception thrown when the D-ID plugin or D-ID service errors. + * + * @public + */ +export class DIDException extends Error { + constructor(message: string) { + super(message); + this.name = 'DIDException'; + } +} + +/** @public */ +export interface JoinSessionTransport { + /** Transport provider. Always `livekit` for this plugin. */ + provider: 'livekit'; + /** LiveKit server URL the D-ID worker should connect to. */ + server_url: string; + /** LiveKit JWT for the D-ID worker. */ + token: string; + /** LiveKit room name to join. */ + room_name: string; +} + +/** @public */ +export interface JoinSessionAudioConfig { + /** Sample rate in Hz. Supported values: 16000, 24000, 48000. */ + sample_rate: number; +} + +/** @public */ +export interface JoinSessionOptions { + /** D-ID agent id. */ + agentId: string; + /** Transport configuration passed to the D-ID join endpoint. */ + transport: JoinSessionTransport; + /** Audio configuration passed to the D-ID join endpoint. */ + audioConfig: JoinSessionAudioConfig; +} + +/** @public */ +export interface DIDAPIOptions { + /** D-ID API key. Falls back to `DID_API_KEY`. */ + apiKey?: string; + /** Override the D-ID API base URL. */ + apiUrl?: string; + /** API retry/timeout options. */ + connOptions?: APIConnectOptions; +} + +/** + * Thin client for the D-ID HTTP API. + * + * @public + */ +export class DIDAPI { + private apiKey: string; + private apiUrl: string; + private connOptions: APIConnectOptions; + + #logger = log(); + + constructor(options: DIDAPIOptions = {}) { + const apiKey = options.apiKey ?? process.env.DID_API_KEY ?? ''; + if (!apiKey) { + throw new DIDException('DID_API_KEY must be set'); + } + + this.apiKey = apiKey; + this.apiUrl = options.apiUrl || DEFAULT_API_URL; + this.connOptions = options.connOptions || DEFAULT_API_CONNECT_OPTIONS; + } + + async joinSession(options: JoinSessionOptions): Promise { + const payload: Record = { + transport: options.transport, + audio_config: options.audioConfig, + }; + + const responseData = (await this.post( + `v2/agents/${options.agentId}/sessions/join`, + payload, + )) as { id: string }; + return responseData.id; + } + + private async post(endpoint: string, payload: Record): Promise { + const url = `${this.apiUrl}/${endpoint}`; + + for (let i = 0; i <= this.connOptions.maxRetry; i++) { + try { + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${this.apiKey}`, + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.connOptions.timeoutMs), + }); + + if (!response.ok) { + const text = await response.text(); + throw new APIStatusError({ + message: 'Server returned an error', + options: { statusCode: response.status, body: { error: text } }, + }); + } + + return await response.json(); + } catch (e) { + if (e instanceof APIStatusError && !e.retryable) { + throw e; + } + if (e instanceof APIConnectionError) { + this.#logger.warn({ error: String(e) }, 'failed to call d-id api'); + } else { + this.#logger.error({ error: e }, 'failed to call d-id api'); + } + + if (i < this.connOptions.maxRetry) { + await new Promise((resolve) => + setTimeout(resolve, intervalForRetry(this.connOptions, i)), + ); + } + } + } + + throw new APIConnectionError({ + message: 'Failed to call D-ID API after all retries', + }); + } +} diff --git a/plugins/did/src/avatar.test.ts b/plugins/did/src/avatar.test.ts new file mode 100644 index 000000000..06e5c5f5d --- /dev/null +++ b/plugins/did/src/avatar.test.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { voice } from '@livekit/agents'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AvatarSession } from './avatar.js'; + +describe('DID AvatarSession', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('calls base AvatarSession.start first', async () => { + const sentinel = new Error('super-start-called'); + const superStartSpy = vi + .spyOn(voice.AvatarSession.prototype, 'start') + .mockRejectedValue(sentinel); + + const avatar = new AvatarSession({ + agentId: 'test-agent-id', + apiKey: 'test-api-key', + }); + + await expect( + avatar.start({ _started: false, output: { audio: null } } as any, {} as any), + ).rejects.toThrow('super-start-called'); + expect(superStartSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/did/src/avatar.ts b/plugins/did/src/avatar.ts new file mode 100644 index 000000000..7906cc47f --- /dev/null +++ b/plugins/did/src/avatar.ts @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + DEFAULT_API_CONNECT_OPTIONS, + getJobContext, + voice, +} from '@livekit/agents'; +import type { Room } from '@livekit/rtc-node'; +import { TrackKind } from '@livekit/rtc-node'; +import type { VideoGrant } from 'livekit-server-sdk'; +import { AccessToken } from 'livekit-server-sdk'; +import { DIDAPI, DIDException } from './api.js'; +import { log } from './log.js'; +import { type AudioConfig, DEFAULT_SAMPLE_RATE } from './types.js'; + +const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf'; +const AVATAR_AGENT_IDENTITY = 'd-id-avatar-agent'; +const AVATAR_AGENT_NAME = 'd-id-avatar-agent'; + +/** + * Options for configuring an AvatarSession. + * + * @public + */ +export interface AvatarSessionOptions { + /** D-ID agent id. Required. See the plugin README for how to create one. */ + agentId: string; + /** Override the D-ID API base URL. */ + apiUrl?: string; + /** D-ID API key. Falls back to `DID_API_KEY`. */ + apiKey?: string; + /** Audio configuration for the stream sent to the D-ID avatar. */ + audioConfig?: AudioConfig; + /** Identity for the avatar participant. Defaults to `d-id-avatar-agent`. */ + avatarParticipantIdentity?: string; + /** Display name for the avatar participant. Defaults to `d-id-avatar-agent`. */ + avatarParticipantName?: string; + /** API retry/timeout options. */ + connOptions?: APIConnectOptions; +} + +/** + * Optional LiveKit credentials for {@link AvatarSession.start}; falls back to env vars. + * + * @public + */ +export interface StartOptions { + /** LiveKit server URL. Falls back to `LIVEKIT_URL`. */ + livekitUrl?: string; + /** LiveKit API key. Falls back to `LIVEKIT_API_KEY`. */ + livekitApiKey?: string; + /** LiveKit API secret. Falls back to `LIVEKIT_API_SECRET`. */ + livekitApiSecret?: string; +} + +/** + * A D-ID avatar session. + * + * @public + */ +export class AvatarSession extends voice.AvatarSession { + private agentId: string; + private audioConfig: AudioConfig; + private avatarParticipantIdentity: string; + private avatarParticipantName: string; + private api: DIDAPI; + + sessionId: string | null = null; + + #logger = log(); + + constructor(options: AvatarSessionOptions) { + super(); + if (!options.agentId) { + throw new DIDException('agentId is required'); + } + this.agentId = options.agentId; + this.audioConfig = options.audioConfig ?? {}; + this.avatarParticipantIdentity = options.avatarParticipantIdentity || AVATAR_AGENT_IDENTITY; + this.avatarParticipantName = options.avatarParticipantName || AVATAR_AGENT_NAME; + this.api = new DIDAPI({ + apiUrl: options.apiUrl, + apiKey: options.apiKey, + connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS, + }); + } + + override get avatarIdentity(): string { + return this.avatarParticipantIdentity; + } + + override get provider(): string { + return 'd-id'; + } + + async start( + agentSession: voice.AgentSession, + room: Room, + options: StartOptions = {}, + ): Promise { + await super.start(agentSession, room); + + const livekitUrl = options.livekitUrl || process.env.LIVEKIT_URL; + const livekitApiKey = options.livekitApiKey || process.env.LIVEKIT_API_KEY; + const livekitApiSecret = options.livekitApiSecret || process.env.LIVEKIT_API_SECRET; + if (!livekitUrl || !livekitApiKey || !livekitApiSecret) { + throw new DIDException( + 'livekitUrl, livekitApiKey, and livekitApiSecret must be set ' + + 'by arguments or environment variables', + ); + } + + let localParticipantIdentity: string; + try { + const jobCtx = getJobContext(); + localParticipantIdentity = jobCtx.agent?.identity || ''; + if (!localParticipantIdentity && room.localParticipant) { + localParticipantIdentity = room.localParticipant.identity; + } + } catch { + if (!room.isConnected || !room.localParticipant) { + throw new DIDException('failed to get local participant identity'); + } + localParticipantIdentity = room.localParticipant.identity; + } + + if (!localParticipantIdentity) { + throw new DIDException('failed to get local participant identity'); + } + + const at = new AccessToken(livekitApiKey, livekitApiSecret, { + identity: this.avatarIdentity, + name: this.avatarParticipantName, + }); + at.kind = 'agent'; + at.addGrant({ + roomJoin: true, + room: room.name, + } as VideoGrant); + at.attributes = { + [ATTRIBUTE_PUBLISH_ON_BEHALF]: localParticipantIdentity, + }; + + const livekitToken = await at.toJwt(); + + const sampleRate = this.audioConfig.sampleRate ?? DEFAULT_SAMPLE_RATE; + + this.#logger.debug('starting avatar session'); + this.sessionId = await this.api.joinSession({ + agentId: this.agentId, + transport: { + provider: 'livekit', + server_url: livekitUrl, + token: livekitToken, + room_name: room.name!, + }, + audioConfig: { sample_rate: sampleRate }, + }); + + agentSession.output.audio = new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }); + } +} diff --git a/plugins/did/src/index.ts b/plugins/did/src/index.ts new file mode 100644 index 000000000..de43f1e30 --- /dev/null +++ b/plugins/did/src/index.ts @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export * from './types.js'; +export * from './api.js'; +export * from './avatar.js'; + +class DIDPlugin extends Plugin { + constructor() { + super({ + title: 'd-id', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new DIDPlugin()); diff --git a/plugins/did/src/log.ts b/plugins/did/src/log.ts new file mode 100644 index 000000000..508b6f807 --- /dev/null +++ b/plugins/did/src/log.ts @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { log as agentsLog } from '@livekit/agents'; +import type { Logger } from 'pino'; + +export const log = (): Logger => agentsLog().child({ plugin: 'did' }); diff --git a/plugins/did/src/types.ts b/plugins/did/src/types.ts new file mode 100644 index 000000000..ff7e6c96b --- /dev/null +++ b/plugins/did/src/types.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** @public */ +export const DEFAULT_SAMPLE_RATE = 24000; + +/** + * Configuration for the audio sent to the D-ID avatar. + * + * @public + */ +export interface AudioConfig { + /** Sample rate in Hz. Supported values: 16000, 24000, 48000. Defaults to 24000. */ + sampleRate?: number; +} diff --git a/plugins/did/tsconfig.json b/plugins/did/tsconfig.json new file mode 100644 index 000000000..acd12bf56 --- /dev/null +++ b/plugins/did/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-did", + "entryPointStrategy": "resolve", + "readme": "none", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/did/tsup.config.ts b/plugins/did/tsup.config.ts new file mode 100644 index 000000000..46011fa8c --- /dev/null +++ b/plugins/did/tsup.config.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { defineConfig } from 'tsup'; +import defaults from '../../tsup.config.js'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50e52a99e..ff61389eb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -257,6 +257,9 @@ importers: '@livekit/agents-plugin-deepgram': specifier: workspace:* version: link:../plugins/deepgram + '@livekit/agents-plugin-did': + specifier: workspace:* + version: link:../plugins/did '@livekit/agents-plugin-elevenlabs': specifier: workspace:* version: link:../plugins/elevenlabs @@ -611,6 +614,31 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/did: + dependencies: + livekit-server-sdk: + specifier: ^2.13.3 + version: 2.14.1 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.29 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + pino: + specifier: ^8.19.0 + version: 8.21.0 + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/elevenlabs: dependencies: '@livekit/mutex': diff --git a/turbo.json b/turbo.json index 325cafca6..c3b1e3ee6 100644 --- a/turbo.json +++ b/turbo.json @@ -18,6 +18,9 @@ "CEREBRAS_API_KEY", "DEEPGRAM_API_KEY", "DEEPSEEK_API_KEY", + "DID_API_KEY", + "DID_API_URL", + "DID_AGENT_ID", "ELEVEN_API_KEY", "FIREWORKS_API_KEY", "FISH_API_KEY",