diff --git a/.changeset/required-judge-tool-choice.md b/.changeset/required-judge-tool-choice.md new file mode 100644 index 000000000..f16afc13b --- /dev/null +++ b/.changeset/required-judge-tool-choice.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Use required tool choice for message judge LLM calls. diff --git a/agents/src/voice/testing/run_result.test.ts b/agents/src/voice/testing/run_result.test.ts index 60e1f27f2..0d31e178c 100644 --- a/agents/src/voice/testing/run_result.test.ts +++ b/agents/src/voice/testing/run_result.test.ts @@ -4,8 +4,15 @@ import { ReadableStream } from 'node:stream/web'; import { describe, expect, it } from 'vitest'; import { z } from 'zod'; -import { FunctionCall } from '../../llm/chat_context.js'; -import { ToolContext, tool } from '../../llm/tool_context.js'; +import { type ChatContext, ChatMessage, FunctionCall } from '../../llm/chat_context.js'; +import { type ChatChunk, LLM, LLMStream } from '../../llm/llm.js'; +import { + type ToolChoice, + ToolContext, + type ToolContextLike, + tool, +} from '../../llm/tool_context.js'; +import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js'; import { Agent } from '../agent.js'; import { performToolExecutions } from '../generation.js'; import { SpeechHandle } from '../speech_handle.js'; @@ -23,6 +30,66 @@ class AgentB extends Agent { } } +class CapturingLLM extends LLM { + toolChoice?: ToolChoice; + + constructor(private readonly toolCall: FunctionCall) { + super(); + } + + label(): string { + return 'capturing'; + } + + chat({ + chatCtx, + toolCtx, + connOptions = DEFAULT_API_CONNECT_OPTIONS, + toolChoice, + }: Parameters[0]): LLMStream { + this.toolChoice = toolChoice; + return new CapturingStream(this, { + chatCtx, + toolCtx, + connOptions, + toolCall: this.toolCall, + }); + } +} + +class CapturingStream extends LLMStream { + constructor( + llm: LLM, + { + chatCtx, + toolCtx, + connOptions, + toolCall, + }: { + chatCtx: ChatContext; + toolCtx?: ToolContextLike; + connOptions: APIConnectOptions; + toolCall: FunctionCall; + }, + ) { + super(llm, { chatCtx, toolCtx, connOptions }); + this.toolCall = toolCall; + } + + private readonly toolCall: FunctionCall; + + protected async run(): Promise { + const chunk: ChatChunk = { + id: 'test', + delta: { + role: 'assistant', + toolCalls: [this.toolCall], + }, + }; + this.queue.put(chunk); + } +} + describe('withMockTools', () => { it('sets the mock registry for the given agent inside the block', () => { const mock = () => 'mocked'; @@ -205,3 +272,26 @@ describe('RunResult speech handle error propagation', () => { await expect(run.wait()).resolves.toBe(run); }); }); + +describe('MessageAssert judge', () => { + it('uses required tool choice', async () => { + const llmInstance = new CapturingLLM( + FunctionCall.create({ + callId: 'call_1', + name: 'check_intent', + args: JSON.stringify({ success: true, reason: 'ok' }), + }), + ); + const result = new RunResult(); + result.events.push({ + type: 'message', + item: ChatMessage.create({ role: 'assistant', content: 'Hello there' }), + }); + + await result.expect.at(0).isMessage({ role: 'assistant' }).judge(llmInstance, { + intent: 'greets the user', + }); + + expect(llmInstance.toolChoice).toBe('required'); + }); +}); diff --git a/agents/src/voice/testing/run_result.ts b/agents/src/voice/testing/run_result.ts index e43e63502..a23e28757 100644 --- a/agents/src/voice/testing/run_result.ts +++ b/agents/src/voice/testing/run_result.ts @@ -938,7 +938,7 @@ export class MessageAssert extends EventAssert { const stream = llm.chat({ chatCtx, toolCtx: [checkIntentTool], - toolChoice: { type: 'function', function: { name: 'check_intent' } }, + toolChoice: 'required', extraKwargs: { temperature: 0 }, });