diff --git a/.changeset/expose-realtime-request-ids.md b/.changeset/expose-realtime-request-ids.md new file mode 100644 index 000000000..1b74e2532 --- /dev/null +++ b/.changeset/expose-realtime-request-ids.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Expose realtime provider response IDs on assistant message metrics as `providerRequestIds`. diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index dbb733a94..595d14e41 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -271,6 +271,13 @@ export function createAudioContent(params: { } export interface MetricsReport { + /** + * Provider-known request or response IDs associated with this turn. + * + * Assistant `ChatMessage` only. These IDs can be used to correlate a turn with + * provider-side logs. + */ + providerRequestIds?: string[]; startedSpeakingAt?: number; stoppedSpeakingAt?: number; transcriptionDelay?: number; diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 6f7998979..fc742c2b7 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -3661,12 +3661,18 @@ export class AgentActivity implements RecognitionHooks { traceTextParts.push(forwardedText); if (addToChatCtx) { + const assistantMetrics: MetricsReport = {}; + if (ev.responseId) { + assistantMetrics.providerRequestIds = [ev.responseId]; + } + const message = ChatMessage.create({ role: 'assistant', content: forwardedText, id: output.message.messageId, interrupted, createdAt: startedSpeakingAt, + metrics: assistantMetrics, }); this.agent._chatCtx.insert(message); speechHandle._itemAdded([message]); diff --git a/agents/src/voice/realtime_message_metrics.test.ts b/agents/src/voice/realtime_message_metrics.test.ts new file mode 100644 index 000000000..e97e4327f --- /dev/null +++ b/agents/src/voice/realtime_message_metrics.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { describe, expect, it } from 'vitest'; +import { ChatContext, type FunctionCall } from '../llm/chat_context.js'; +import { + type GenerationCreatedEvent, + type MessageGeneration, + type RealtimeCapabilities, + RealtimeModel, + RealtimeSession, +} from '../llm/realtime.js'; +import { type ToolChoice, ToolContext } from '../llm/tool_context.js'; +import { initializeLogger } from '../log.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes, type ConversationItemAddedEvent } from './events.js'; + +initializeLogger({ pretty: false, level: 'silent' }); + +function emptyStream(): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); +} + +function oneItemStream(item: T): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(item); + controller.close(); + }, + }); +} + +class FakeRealtimeSession extends RealtimeSession { + private _chatCtx = ChatContext.empty(); + private _tools = ToolContext.empty(); + + get chatCtx(): ChatContext { + return this._chatCtx; + } + + get tools(): ToolContext { + return this._tools; + } + + async updateInstructions(_instructions: string): Promise {} + + async updateChatCtx(chatCtx: ChatContext): Promise { + this._chatCtx = chatCtx.copy(); + } + + async updateTools(tools: ToolContext): Promise { + this._tools = tools.copy(); + } + + updateOptions(_options: { toolChoice?: ToolChoice | null }): void {} + + pushAudio(_frame: AudioFrame): void {} + + async generateReply(): Promise { + const message: MessageGeneration = { + messageId: 'message-id', + textStream: oneItemStream('Hello'), + audioStream: emptyStream(), + modalities: Promise.resolve(['text']), + }; + + return { + messageStream: oneItemStream(message), + functionStream: emptyStream(), + userInitiated: true, + responseId: 'provider-response-id', + }; + } + + async commitAudio(): Promise {} + + async clearAudio(): Promise {} + + async interrupt(): Promise {} + + async truncate(): Promise {} +} + +class FakeRealtimeModel extends RealtimeModel { + readonly activeSession: FakeRealtimeSession; + + constructor() { + const capabilities: RealtimeCapabilities = { + messageTruncation: false, + turnDetection: false, + userTranscription: false, + autoToolReplyGeneration: false, + audioOutput: false, + manualFunctionCalls: false, + midSessionChatCtxUpdate: true, + midSessionInstructionsUpdate: true, + midSessionToolsUpdate: true, + perResponseToolChoice: false, + }; + super(capabilities); + this.activeSession = new FakeRealtimeSession(this); + } + + get model(): string { + return 'fake-realtime'; + } + + session(): RealtimeSession { + return this.activeSession; + } + + async close(): Promise {} +} + +describe('Realtime message metrics', () => { + it('makes realtime response IDs available on assistant messages', async () => { + const llm = new FakeRealtimeModel(); + const session = new AgentSession({ llm, vad: null, turnHandling: { turnDetection: null } }); + const conversationEvents: ConversationItemAddedEvent[] = []; + + session.on(AgentSessionEventTypes.ConversationItemAdded, (ev) => { + conversationEvents.push(ev); + }); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + try { + await session.generateReply().waitForPlayout(); + } finally { + await session.close(); + } + + const assistantMessages = conversationEvents + .map((event) => event.item) + .filter((item) => item.type === 'message' && item.role === 'assistant'); + + expect(assistantMessages).toHaveLength(1); + expect(assistantMessages[0]?.metrics.providerRequestIds).toEqual(['provider-response-id']); + }); +});