From 7e1d71c3f87fdc5e5ba022f8d50941d5fb6e89e4 Mon Sep 17 00:00:00 2001 From: Toubat Date: Thu, 6 Nov 2025 18:02:03 +0800 Subject: [PATCH 1/2] save --- agents/src/llm/chat_context.ts | 54 +++++++++++++- agents/src/llm/index.ts | 1 + agents/src/llm/provider_format/google.test.ts | 73 ++++++++++++++++++- agents/src/llm/provider_format/openai.test.ts | 56 +++++++++++++- agents/src/llm/provider_format/openai.ts | 5 +- agents/src/voice/agent.ts | 22 ++++++ agents/src/voice/agent_session.ts | 13 +++- 7 files changed, 218 insertions(+), 6 deletions(-) diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 138959490..cddd08d62 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -300,7 +300,59 @@ export class FunctionCallOutput { } } -export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput; +export class AgentHandoffItem { + readonly id: string; + + readonly type = 'agent_handoff' as const; + + oldAgentId: string | undefined; + + newAgentId: string; + + createdAt: number; + + constructor(params: { + oldAgentId?: string; + newAgentId: string; + id?: string; + createdAt?: number; + }) { + const { oldAgentId, newAgentId, id = shortuuid('item_'), createdAt = Date.now() } = params; + this.id = id; + this.oldAgentId = oldAgentId; + this.newAgentId = newAgentId; + this.createdAt = createdAt; + } + + static create(params: { + oldAgentId?: string; + newAgentId: string; + id?: string; + createdAt?: number; + }) { + return new AgentHandoffItem(params); + } + + toJSON(excludeTimestamp: boolean = false): JSONValue { + const result: JSONValue = { + id: this.id, + type: this.type, + newAgentId: this.newAgentId, + }; + + if (this.oldAgentId !== undefined) { + result.oldAgentId = this.oldAgentId; + } + + if (!excludeTimestamp) { + result.createdAt = this.createdAt; + } + + return result; + } +} + +export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput | AgentHandoffItem; export class ChatContext { protected _items: ChatItem[]; diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 45231c78e..b33d5d64d 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -17,6 +17,7 @@ export { } from './tool_context.js'; export { + AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, diff --git a/agents/src/llm/provider_format/google.test.ts b/agents/src/llm/provider_format/google.test.ts index 895b3bbc9..a86d8460b 100644 --- a/agents/src/llm/provider_format/google.test.ts +++ b/agents/src/llm/provider_format/google.test.ts @@ -4,7 +4,12 @@ import { VideoBufferType, VideoFrame } from '@livekit/rtc-node'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { initializeLogger } from '../../log.js'; -import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js'; +import { + AgentHandoffItem, + ChatContext, + FunctionCall, + FunctionCallOutput, +} from '../chat_context.js'; import { serializeImage } from '../utils.js'; import { toChatCtx } from './google.js'; @@ -769,4 +774,70 @@ describe('Google Provider Format - toChatCtx', () => { ]); expect(formatData.systemMessages).toBeNull(); }); + + it('should filter out agent handoff items', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Hello' }); + + // Insert an agent handoff item + const handoff = new AgentHandoffItem({ + oldAgentId: 'agent_1', + newAgentId: 'agent_2', + }); + ctx.insert(handoff); + + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + // Agent handoff should be filtered out, only messages should remain + expect(result).toEqual([ + { + role: 'user', + parts: [{ text: 'Hello' }], + }, + { + role: 'model', + parts: [{ text: 'Hi there!' }], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should handle multiple agent handoffs without errors', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Start' }); + + // Multiple handoffs + ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' }); + + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' }); + + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + // All handoffs should be filtered out + // Note: Google provider groups consecutive messages by the same role + expect(result).toEqual([ + { + role: 'user', + parts: [{ text: 'Start' }], + }, + { + role: 'model', + parts: [ + { text: 'Response from agent 1' }, + { text: 'Response from agent 2' }, + { text: 'Response from agent 3' }, + ], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); }); diff --git a/agents/src/llm/provider_format/openai.test.ts b/agents/src/llm/provider_format/openai.test.ts index 655355dc4..6db565319 100644 --- a/agents/src/llm/provider_format/openai.test.ts +++ b/agents/src/llm/provider_format/openai.test.ts @@ -4,7 +4,12 @@ import { VideoBufferType, VideoFrame } from '@livekit/rtc-node'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { initializeLogger } from '../../log.js'; -import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js'; +import { + AgentHandoffItem, + ChatContext, + FunctionCall, + FunctionCallOutput, +} from '../chat_context.js'; import { serializeImage } from '../utils.js'; import { toChatCtx } from './openai.js'; @@ -578,4 +583,53 @@ describe('toChatCtx', () => { }, ]); }); + + it('should filter out agent handoff items', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Hello' }); + + // Insert an agent handoff item + const handoff = new AgentHandoffItem({ + oldAgentId: 'agent_1', + newAgentId: 'agent_2', + }); + ctx.insert(handoff); + + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const result = await toChatCtx(ctx); + + // Agent handoff should be filtered out, only messages should remain + expect(result).toEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]); + }); + + it('should handle multiple agent handoffs without errors', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Start' }); + + // Multiple handoffs + ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' }); + + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' }); + + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' })); + ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' }); + + const result = await toChatCtx(ctx); + + // All handoffs should be filtered out + expect(result).toEqual([ + { role: 'user', content: 'Start' }, + { role: 'assistant', content: 'Response from agent 1' }, + { role: 'assistant', content: 'Response from agent 2' }, + { role: 'assistant', content: 'Response from agent 3' }, + ]); + }); }); diff --git a/agents/src/llm/provider_format/openai.ts b/agents/src/llm/provider_format/openai.ts index ed89c6e5a..3d0b77d98 100644 --- a/agents/src/llm/provider_format/openai.ts +++ b/agents/src/llm/provider_format/openai.ts @@ -78,9 +78,10 @@ async function toChatItem(item: ChatItem) { tool_call_id: item.callId, content: item.output, }; - } else { - throw new Error(`Unsupported item type: ${item['type']}`); } + // Skip other item types (e.g., agent_handoff) + // These should be filtered by groupToolCalls, but this is a safety net + throw new Error(`Unsupported item type: ${item['type']}`); } async function toImageContent(content: ImageContent) { diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 29b98b306..6f529f038 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -59,6 +59,7 @@ export interface ModelSettings { } export interface AgentOptions { + id?: string; instructions: string; chatCtx?: ChatContext; tools?: ToolContext; @@ -72,6 +73,7 @@ export interface AgentOptions { } export class Agent { + private _id: string; private turnDetection?: TurnDetectionMode; private _stt?: STT; private _vad?: VAD; @@ -91,6 +93,7 @@ export class Agent { _tools?: ToolContext; constructor({ + id, instructions, chatCtx, tools, @@ -100,6 +103,21 @@ export class Agent { llm, tts, }: AgentOptions) { + if (id) { + this._id = id; + } else { + // Convert class name to snake_case + const className = this.constructor.name; + if (className === 'Agent') { + this._id = 'default_agent'; + } else { + this._id = className + .replace(/([A-Z])/g, '_$1') + .toLowerCase() + .replace(/^_/, ''); + } + } + this._instructions = instructions; this._tools = { ...tools }; this._chatCtx = chatCtx @@ -152,6 +170,10 @@ export class Agent { return new ReadonlyChatContext(this._chatCtx.items); } + get id(): string { + return this._id; + } + get instructions(): string { return this._instructions; } diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 758d116c4..e538c236b 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -14,7 +14,7 @@ import { type TTSModelString, } from '../inference/index.js'; import { getJobContext } from '../job.js'; -import { ChatContext, ChatMessage } from '../llm/chat_context.js'; +import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js'; import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; import { log } from '../log.js'; @@ -336,6 +336,8 @@ export class AgentSession< // TODO(AJS-129): add lock to agent activity core lifecycle this.nextActivity = new AgentActivity(agent, this); + const previousActivity = this.activity; + if (this.activity) { await this.activity.drain(); await this.activity.close(); @@ -344,6 +346,15 @@ export class AgentSession< this.activity = this.nextActivity; this.nextActivity = undefined; + // Insert agent handoff into chat context + this._chatCtx.insert( + new AgentHandoffItem({ + oldAgentId: previousActivity?.agent.id, + newAgentId: agent.id, + }), + ); + this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context'); + await this.activity.start(); if (this._input.audio) { From d57c81776afe5a7a6510e49df66ae2d79de513bc Mon Sep 17 00:00:00 2001 From: Toubat Date: Fri, 7 Nov 2025 16:39:57 +0800 Subject: [PATCH 2/2] Update agent_session.ts --- agents/src/voice/agent_session.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 6bf12b194..873d69c84 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -372,7 +372,6 @@ export class AgentSession< this.activity = this.nextActivity; this.nextActivity = undefined; - // Insert agent handoff into chat context this._chatCtx.insert( new AgentHandoffItem({ oldAgentId: previousActivity?.agent.id,