From 7b34b9bc8623c836aed95b1a83a9e76b219af109 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 06:43:09 +0000 Subject: [PATCH 1/2] feat(voice): expose realtime provider request IDs --- .changeset/expose-realtime-request-ids.md | 5 + agents/src/llm/chat_context.ts | 7 + agents/src/voice/agent_activity.ts | 6 + .../voice/realtime_message_metrics.test.ts | 146 ++++++++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 .changeset/expose-realtime-request-ids.md create mode 100644 agents/src/voice/realtime_message_metrics.test.ts diff --git a/.changeset/expose-realtime-request-ids.md b/.changeset/expose-realtime-request-ids.md new file mode 100644 index 000000000..77751858d --- /dev/null +++ b/.changeset/expose-realtime-request-ids.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': minor +--- + +Expose realtime provider response IDs on assistant message metrics as `providerRequestIds`. diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index dbb733a94..595d14e41 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -271,6 +271,13 @@ export function createAudioContent(params: { } export interface MetricsReport { + /** + * Provider-known request or response IDs associated with this turn. + * + * Assistant `ChatMessage` only. These IDs can be used to correlate a turn with + * provider-side logs. + */ + providerRequestIds?: string[]; startedSpeakingAt?: number; stoppedSpeakingAt?: number; transcriptionDelay?: number; diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 6f7998979..fc742c2b7 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -3661,12 +3661,18 @@ export class AgentActivity implements RecognitionHooks { traceTextParts.push(forwardedText); if (addToChatCtx) { + const assistantMetrics: MetricsReport = {}; + if (ev.responseId) { + assistantMetrics.providerRequestIds = [ev.responseId]; + } + const message = ChatMessage.create({ role: 'assistant', content: forwardedText, id: output.message.messageId, interrupted, createdAt: startedSpeakingAt, + metrics: assistantMetrics, }); this.agent._chatCtx.insert(message); speechHandle._itemAdded([message]); diff --git a/agents/src/voice/realtime_message_metrics.test.ts b/agents/src/voice/realtime_message_metrics.test.ts new file mode 100644 index 000000000..e97e4327f --- /dev/null +++ b/agents/src/voice/realtime_message_metrics.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { describe, expect, it } from 'vitest'; +import { ChatContext, type FunctionCall } from '../llm/chat_context.js'; +import { + type GenerationCreatedEvent, + type MessageGeneration, + type RealtimeCapabilities, + RealtimeModel, + RealtimeSession, +} from '../llm/realtime.js'; +import { type ToolChoice, ToolContext } from '../llm/tool_context.js'; +import { initializeLogger } from '../log.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes, type ConversationItemAddedEvent } from './events.js'; + +initializeLogger({ pretty: false, level: 'silent' }); + +function emptyStream(): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); +} + +function oneItemStream(item: T): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(item); + controller.close(); + }, + }); +} + +class FakeRealtimeSession extends RealtimeSession { + private _chatCtx = ChatContext.empty(); + private _tools = ToolContext.empty(); + + get chatCtx(): ChatContext { + return this._chatCtx; + } + + get tools(): ToolContext { + return this._tools; + } + + async updateInstructions(_instructions: string): Promise {} + + async updateChatCtx(chatCtx: ChatContext): Promise { + this._chatCtx = chatCtx.copy(); + } + + async updateTools(tools: ToolContext): Promise { + this._tools = tools.copy(); + } + + updateOptions(_options: { toolChoice?: ToolChoice | null }): void {} + + pushAudio(_frame: AudioFrame): void {} + + async generateReply(): Promise { + const message: MessageGeneration = { + messageId: 'message-id', + textStream: oneItemStream('Hello'), + audioStream: emptyStream(), + modalities: Promise.resolve(['text']), + }; + + return { + messageStream: oneItemStream(message), + functionStream: emptyStream(), + userInitiated: true, + responseId: 'provider-response-id', + }; + } + + async commitAudio(): Promise {} + + async clearAudio(): Promise {} + + async interrupt(): Promise {} + + async truncate(): Promise {} +} + +class FakeRealtimeModel extends RealtimeModel { + readonly activeSession: FakeRealtimeSession; + + constructor() { + const capabilities: RealtimeCapabilities = { + messageTruncation: false, + turnDetection: false, + userTranscription: false, + autoToolReplyGeneration: false, + audioOutput: false, + manualFunctionCalls: false, + midSessionChatCtxUpdate: true, + midSessionInstructionsUpdate: true, + midSessionToolsUpdate: true, + perResponseToolChoice: false, + }; + super(capabilities); + this.activeSession = new FakeRealtimeSession(this); + } + + get model(): string { + return 'fake-realtime'; + } + + session(): RealtimeSession { + return this.activeSession; + } + + async close(): Promise {} +} + +describe('Realtime message metrics', () => { + it('makes realtime response IDs available on assistant messages', async () => { + const llm = new FakeRealtimeModel(); + const session = new AgentSession({ llm, vad: null, turnHandling: { turnDetection: null } }); + const conversationEvents: ConversationItemAddedEvent[] = []; + + session.on(AgentSessionEventTypes.ConversationItemAdded, (ev) => { + conversationEvents.push(ev); + }); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + try { + await session.generateReply().waitForPlayout(); + } finally { + await session.close(); + } + + const assistantMessages = conversationEvents + .map((event) => event.item) + .filter((item) => item.type === 'message' && item.role === 'assistant'); + + expect(assistantMessages).toHaveLength(1); + expect(assistantMessages[0]?.metrics.providerRequestIds).toEqual(['provider-response-id']); + }); +}); From 53ed4cef4d31778465995068f24ef2733d85052c Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 22 Jul 2026 14:35:33 -0700 Subject: [PATCH 2/2] chore: downgrade provider request IDs changeset to patch Keep the next release as a patch bump rather than minor. Co-authored-by: Cursor --- .changeset/expose-realtime-request-ids.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/expose-realtime-request-ids.md b/.changeset/expose-realtime-request-ids.md index 77751858d..1b74e2532 100644 --- a/.changeset/expose-realtime-request-ids.md +++ b/.changeset/expose-realtime-request-ids.md @@ -1,5 +1,5 @@ --- -'@livekit/agents': minor +'@livekit/agents': patch --- Expose realtime provider response IDs on assistant message metrics as `providerRequestIds`.