diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index 63e103ebd..6658c9ccc 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -339,6 +339,7 @@ export function defineAnthropicInstrumentationAssertions(options: { name: string; snapshotName: string; supportsBetaMessages: boolean; + supportsServerToolUse: boolean; supportsThinking: boolean; testFileUrl: string; timeoutMs: number; @@ -543,6 +544,38 @@ export function defineAnthropicInstrumentationAssertions(options: { }, ); + if (options.supportsServerToolUse) { + test("captures server tool usage metrics", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-server-tool-use-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + const output = span?.output as + | { content?: Array<{ name?: string; type?: string }> } + | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(span?.metrics).toMatchObject({ + server_tool_use_web_search_requests: expect.any(Number), + }); + expect( + output?.content?.some( + (block) => + block.type === "server_tool_use" && block.name === "web_search", + ), + ).toBe(true); + }); + } + if (options.supportsThinking) { test("captures trace for streaming extended thinking", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs index f33a4f984..e59706856 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs @@ -3,5 +3,8 @@ import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runAutoAnthropicInstrumentation(Anthropic, { useBetaMessages: false }), + runAutoAnthropicInstrumentation(Anthropic, { + supportsServerToolUse: false, + useBetaMessages: false, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts index f6d3267ec..baa2d0b53 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts @@ -3,5 +3,8 @@ import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runWrappedAnthropicInstrumentation(Anthropic, { useBetaMessages: false }), + runWrappedAnthropicInstrumentation(Anthropic, { + supportsServerToolUse: false, + useBetaMessages: false, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs index 0fc19dd85..60a00d927 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs @@ -2,4 +2,8 @@ import Anthropic from "anthropic-sdk-v0390"; import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; -runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); +runMain(async () => + runAutoAnthropicInstrumentation(Anthropic, { + supportsServerToolUse: false, + }), +); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts index 47743faae..7c290d5c3 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts @@ -2,4 +2,8 @@ import Anthropic from "anthropic-sdk-v0390"; import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; -runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); +runMain(async () => + runWrappedAnthropicInstrumentation(Anthropic, { + supportsServerToolUse: false, + }), +); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs index bab1c7703..1d0bf8f11 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runAutoAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runAutoAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts index 45e6c2145..1af6b4d0c 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runWrappedAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runWrappedAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs index 5de8bde13..e24e47393 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runAutoAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runAutoAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts index ed9406332..435d4cb3b 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runWrappedAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runWrappedAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs index 03c3989a3..4e77b5201 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runAutoAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runAutoAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts index 35c9aeabb..f9b4e0643 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runWrappedAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runWrappedAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs index 3cc3f4e49..affe4ab3c 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs @@ -23,10 +23,20 @@ const WEATHER_TOOL = { required: ["location"], }, }; +const WEB_SEARCH_SERVER_TOOL = { + type: "web_search_20250305", + name: "web_search", + max_uses: 1, +}; async function runAnthropicInstrumentationScenario( Anthropic, - { decorateClient, useBetaMessages = true, supportsThinking = false } = {}, + { + decorateClient, + useBetaMessages = true, + supportsThinking = false, + supportsServerToolUse = true, + } = {}, ) { const imageBase64 = ( await readFile(new URL("./test-image.png", import.meta.url)) @@ -178,6 +188,29 @@ async function runAnthropicInstrumentationScenario( }); }); + if (supportsServerToolUse) { + await runOperation( + "anthropic-server-tool-use-operation", + "server-tool-use", + async () => { + await client.messages.create({ + model: "claude-sonnet-4-5-20250929", + max_tokens: 256, + temperature: 0, + tools: [WEB_SEARCH_SERVER_TOOL], + tool_choice: { type: "tool", name: WEB_SEARCH_SERVER_TOOL.name }, + messages: [ + { + role: "user", + content: + "Use web_search to find one recent AI SDK release headline and return one short sentence.", + }, + ], + }); + }, + ); + } + if (supportsThinking) { await runOperation( "anthropic-stream-thinking-operation", diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.mjs index 05bced36e..6b8fdf566 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.mjs @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runAutoAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runAutoAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.test.ts b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts index 1b3de5cf3..723d72e74 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.test.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts @@ -17,6 +17,7 @@ const anthropicScenarios = await Promise.all( dependencyName: "anthropic-sdk-v0273", snapshotName: "anthropic-v0273", supportsBetaMessages: false, + supportsServerToolUse: false, supportsThinking: false, wrapperEntry: "scenario.anthropic-v0273.ts", }, @@ -25,6 +26,7 @@ const anthropicScenarios = await Promise.all( dependencyName: "anthropic-sdk-v0390", snapshotName: "anthropic-v0390", supportsBetaMessages: true, + supportsServerToolUse: false, supportsThinking: false, wrapperEntry: "scenario.anthropic-v0390.ts", }, @@ -83,6 +85,7 @@ for (const scenario of anthropicScenarios) { }, snapshotName: scenario.snapshotName, supportsBetaMessages: scenario.supportsBetaMessages, + supportsServerToolUse: scenario.supportsServerToolUse ?? true, supportsThinking: scenario.supportsThinking, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, @@ -101,6 +104,7 @@ for (const scenario of anthropicScenarios) { }, snapshotName: scenario.snapshotName, supportsBetaMessages: scenario.supportsBetaMessages, + supportsServerToolUse: scenario.supportsServerToolUse ?? true, supportsThinking: scenario.supportsThinking, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.ts b/e2e/scenarios/anthropic-instrumentation/scenario.ts index 83a21f6bd..ed72643df 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.ts @@ -3,5 +3,7 @@ import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; runMain(async () => - runWrappedAnthropicInstrumentation(Anthropic, { supportsThinking: true }), + runWrappedAnthropicInstrumentation(Anthropic, { + supportsThinking: true, + }), ); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs index 1ff56ef23..65658b5b7 100644 --- a/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs @@ -9,6 +9,12 @@ import { const GOOGLE_MODEL = "gemini-2.5-flash-lite"; const ROOT_NAME = "google-genai-instrumentation-root"; const SCENARIO_NAME = "google-genai-instrumentation"; +const GOOGLE_GENAI_RETRY_OPTIONS = { + attempts: 3, + delayMs: 1_000, + maxDelayMs: 5_000, + shouldRetry: isRetriableGoogleGenAIError, +}; const WEATHER_TOOL = { functionDeclarations: [ { @@ -28,6 +34,72 @@ const WEATHER_TOOL = { ], }; +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function getRetryStatus(error) { + if (!isObject(error)) { + return undefined; + } + + const directStatus = error.status; + if (typeof directStatus === "number") { + return directStatus; + } + + const nestedError = error.error; + if ( + isObject(nestedError) && + typeof nestedError.code === "number" && + Number.isFinite(nestedError.code) + ) { + return nestedError.code; + } + + return undefined; +} + +function isRetriableGoogleGenAIError(error) { + const status = getRetryStatus(error); + if (status === 429 || status === 500 || status === 503 || status === 504) { + return true; + } + + const message = error instanceof Error ? error.message : String(error ?? ""); + return ( + message.includes("request timed out") || + message.includes("UNAVAILABLE") || + message.includes("temporarily unavailable") + ); +} + +async function withRetry( + callback, + { + attempts = 3, + delayMs = 1_000, + maxDelayMs = Number.POSITIVE_INFINITY, + shouldRetry = () => true, + } = {}, +) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await callback(); + } catch (error) { + lastError = error; + if (attempt === attempts || !shouldRetry(error)) { + throw error; + } + const retryDelayMs = Math.min(delayMs * attempt, maxDelayMs); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + } + + throw lastError; +} + async function runGoogleGenAIInstrumentationScenario(sdk, options = {}) { const imageBase64 = ( await readFile(new URL("./test-image.png", import.meta.url)) @@ -41,94 +113,104 @@ async function runGoogleGenAIInstrumentationScenario(sdk, options = {}) { await runTracedScenario({ callback: async () => { await runOperation("google-generate-operation", "generate", async () => { - await client.models.generateContent({ - model: GOOGLE_MODEL, - contents: "Reply with exactly PARIS.", - config: { - maxOutputTokens: 24, - temperature: 0, - }, - }); + await withRetry(async () => { + await client.models.generateContent({ + model: GOOGLE_MODEL, + contents: "Reply with exactly PARIS.", + config: { + maxOutputTokens: 24, + temperature: 0, + }, + }); + }, GOOGLE_GENAI_RETRY_OPTIONS); }); await runOperation( "google-attachment-operation", "attachment", async () => { - await client.models.generateContent({ - model: GOOGLE_MODEL, - contents: [ - { - parts: [ - { - inlineData: { - data: imageBase64, - mimeType: "image/png", + await withRetry(async () => { + await client.models.generateContent({ + model: GOOGLE_MODEL, + contents: [ + { + parts: [ + { + inlineData: { + data: imageBase64, + mimeType: "image/png", + }, + }, + { + text: "Describe the attached image in one short sentence.", }, - }, - { - text: "Describe the attached image in one short sentence.", - }, - ], - role: "user", + ], + role: "user", + }, + ], + config: { + maxOutputTokens: 48, + temperature: 0, }, - ], - config: { - maxOutputTokens: 48, - temperature: 0, - }, - }); + }); + }, GOOGLE_GENAI_RETRY_OPTIONS); }, ); await runOperation("google-stream-operation", "stream", async () => { - const stream = await client.models.generateContentStream({ - model: GOOGLE_MODEL, - contents: "Count from 1 to 3 and include the words one two three.", - config: { - maxOutputTokens: 64, - temperature: 0, - }, - }); - await collectAsync(stream); - }); - - await runOperation( - "google-stream-return-operation", - "stream-return", - async () => { + await withRetry(async () => { const stream = await client.models.generateContentStream({ model: GOOGLE_MODEL, - contents: "Reply with exactly BONJOUR.", + contents: "Count from 1 to 3 and include the words one two three.", config: { - maxOutputTokens: 24, + maxOutputTokens: 64, temperature: 0, }, }); + await collectAsync(stream); + }, GOOGLE_GENAI_RETRY_OPTIONS); + }); - for await (const _chunk of stream) { - break; - } + await runOperation( + "google-stream-return-operation", + "stream-return", + async () => { + await withRetry(async () => { + const stream = await client.models.generateContentStream({ + model: GOOGLE_MODEL, + contents: "Reply with exactly BONJOUR.", + config: { + maxOutputTokens: 24, + temperature: 0, + }, + }); + + for await (const _chunk of stream) { + break; + } + }, GOOGLE_GENAI_RETRY_OPTIONS); }, ); await runOperation("google-tool-operation", "tool", async () => { - await client.models.generateContent({ - model: GOOGLE_MODEL, - contents: - "Use the get_weather function for Paris, France. Do not answer from memory.", - config: { - maxOutputTokens: 128, - temperature: 0, - tools: [WEATHER_TOOL], - toolConfig: { - functionCallingConfig: { - allowedFunctionNames: ["get_weather"], - mode: sdk.FunctionCallingConfigMode.ANY, + await withRetry(async () => { + await client.models.generateContent({ + model: GOOGLE_MODEL, + contents: + "Use the get_weather function for Paris, France. Do not answer from memory.", + config: { + maxOutputTokens: 128, + temperature: 0, + tools: [WEATHER_TOOL], + toolConfig: { + functionCallingConfig: { + allowedFunctionNames: ["get_weather"], + mode: sdk.FunctionCallingConfigMode.ANY, + }, }, }, - }, - }); + }); + }, GOOGLE_GENAI_RETRY_OPTIONS); }); }, metadata: { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index 35112328e..fd9eb037a 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -189,6 +189,45 @@ describe("parseMetricsFromUsage", () => { it("should handle empty usage object", () => { expect(parseMetricsFromUsage({})).toEqual({}); }); + + it("should flatten server_tool_use metrics", () => { + const usage = { + input_tokens: 100, + output_tokens: 50, + server_tool_use: { + web_search_requests: 3, + ignored: "not-a-number", + }, + }; + + const result = parseMetricsFromUsage(usage); + + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + server_tool_use_web_search_requests: 3, + }); + }); + + it("should flatten web_fetch and tool_search server tool metrics", () => { + const usage = { + input_tokens: 100, + output_tokens: 50, + server_tool_use: { + web_fetch_requests: 2, + tool_search_requests: 4, + }, + }; + + const result = parseMetricsFromUsage(usage); + + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + server_tool_use_web_fetch_requests: 2, + server_tool_use_tool_search_requests: 4, + }); + }); }); describe("aggregateAnthropicStreamChunks", () => { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 3632c0b80..54c056a6a 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -124,6 +124,14 @@ export function parseMetricsFromUsage( saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens"); saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens"); + if (isObject(usage.server_tool_use)) { + for (const [name, value] of Object.entries(usage.server_tool_use)) { + if (typeof value === "number") { + metrics[`server_tool_use_${name}`] = value; + } + } + } + return metrics; } diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index d661471c1..79f8258b1 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -103,6 +103,13 @@ export interface AnthropicUsage { output_tokens: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number; + server_tool_use?: AnthropicServerToolUseUsage; + [key: string]: unknown; +} + +export interface AnthropicServerToolUseUsage { + web_search_requests?: number; + [key: string]: unknown; } export type AnthropicStreamEvent =