From f26ffa2bdb2767607cdb75c7a37a45e7f147ec34 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 13 May 2026 17:11:47 +0000 Subject: [PATCH 1/2] feat: instrument telemetry for invoke command --- .../commands/invoke/__tests__/invoke.test.ts | 91 ++++++++- src/cli/commands/invoke/command.tsx | 186 +++++++++++------- 2 files changed, 198 insertions(+), 79 deletions(-) diff --git a/src/cli/commands/invoke/__tests__/invoke.test.ts b/src/cli/commands/invoke/__tests__/invoke.test.ts index 6414a756e..1610fea33 100644 --- a/src/cli/commands/invoke/__tests__/invoke.test.ts +++ b/src/cli/commands/invoke/__tests__/invoke.test.ts @@ -1,13 +1,14 @@ -import { runCLI } from '../../../../test-utils/index.js'; +import { createTelemetryHelper, runCLI } from '../../../../test-utils/index.js'; import { randomUUID } from 'node:crypto'; import { mkdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; describe('invoke command', () => { let testDir: string; let projectDir: string; + const telemetry = createTelemetryHelper(); beforeAll(async () => { testDir = join(tmpdir(), `agentcore-invoke-${randomUUID()}`); @@ -45,23 +46,37 @@ describe('invoke command', () => { } }); + afterEach(() => { + telemetry.clearEntries(); + }); + afterAll(async () => { + telemetry.destroy(); await rm(testDir, { recursive: true, force: true }); }); describe('validation', () => { it('requires prompt for JSON output', async () => { - const result = await runCLI(['invoke', '--json'], projectDir); + const result = await runCLI(['invoke', '--json'], projectDir, { env: telemetry.env }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); expect(json.error.includes('Prompt'), `Error should mention Prompt: ${json.error}`).toBeTruthy(); + telemetry.assertMetricEmitted({ + command: 'invoke', + exit_reason: 'failure', + has_stream: false, + has_session_id: false, + auth_type: 'sigv4', + }); }); }); describe('agent validation', () => { it('rejects non-existent agent', async () => { - const result = await runCLI(['invoke', 'hello', '--runtime', 'nonexistent', '--json'], projectDir); + const result = await runCLI(['invoke', 'hello', '--runtime', 'nonexistent', '--json'], projectDir, { + env: telemetry.env, + }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); @@ -69,28 +84,50 @@ describe('invoke command', () => { json.error.includes('not found') || json.error.includes('No deployed'), `Error should mention not found: ${json.error}` ).toBeTruthy(); + telemetry.assertMetricEmitted({ + command: 'invoke', + exit_reason: 'failure', + protocol: 'http', + auth_type: 'sigv4', + has_session_id: false, + }); }); }); describe('streaming', () => { it('command accepts --stream flag', async () => { - const result = await runCLI(['invoke', 'hello', '--stream', '--json'], projectDir); + const result = await runCLI(['invoke', 'hello', '--stream', '--json'], projectDir, { env: telemetry.env }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); // Should fail because not deployed, not because of invalid flags + telemetry.assertMetricEmitted({ + command: 'invoke', + has_stream: true, + exit_reason: 'failure', + auth_type: 'sigv4', + has_session_id: false, + }); }); it('--stream with invalid agent returns error', async () => { - const result = await runCLI(['invoke', 'hello', '--stream', '--runtime', 'nonexistent', '--json'], projectDir); + const result = await runCLI(['invoke', 'hello', '--stream', '--runtime', 'nonexistent', '--json'], projectDir, { + env: telemetry.env, + }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); expect(json.error.length > 0, 'Should have error message').toBeTruthy(); + telemetry.assertMetricEmitted({ + command: 'invoke', + has_stream: true, + exit_reason: 'failure', + auth_type: 'sigv4', + }); }); it('requires prompt for streaming', async () => { - const result = await runCLI(['invoke', '--stream', '--json'], projectDir); + const result = await runCLI(['invoke', '--stream', '--json'], projectDir, { env: telemetry.env }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); @@ -98,6 +135,46 @@ describe('invoke command', () => { json.error.toLowerCase().includes('prompt') || json.error.toLowerCase().includes('deploy'), `Error should mention prompt or deployment: ${json.error}` ).toBeTruthy(); + telemetry.assertMetricEmitted({ + command: 'invoke', + has_stream: true, + exit_reason: 'failure', + auth_type: 'sigv4', + has_session_id: false, + }); + }); + }); + + describe('bearer token auth', () => { + it('records auth_type bearer_token', async () => { + const result = await runCLI(['invoke', 'hello', '--bearer-token', 'fake-token', '--json'], projectDir, { + env: telemetry.env, + }); + expect(result.exitCode).toBe(1); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(false); + telemetry.assertMetricEmitted({ + command: 'invoke', + auth_type: 'bearer_token', + exit_reason: 'failure', + }); + }); + }); + + describe('session id', () => { + it('records has_session_id true', async () => { + const result = await runCLI(['invoke', 'hello', '--session-id', 'test-session', '--json'], projectDir, { + env: telemetry.env, + }); + expect(result.exitCode).toBe(1); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(false); + telemetry.assertMetricEmitted({ + command: 'invoke', + has_session_id: true, + exit_reason: 'failure', + auth_type: 'sigv4', + }); }); }); }); diff --git a/src/cli/commands/invoke/command.tsx b/src/cli/commands/invoke/command.tsx index 00d25a7d4..2ec2e5b10 100644 --- a/src/cli/commands/invoke/command.tsx +++ b/src/cli/commands/invoke/command.tsx @@ -1,12 +1,14 @@ -import { serializeResult } from '../../../lib'; +import { type Result, serializeResult } from '../../../lib'; import { getErrorMessage } from '../../errors'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; +import { AuthType, Protocol, standardize } from '../../telemetry/schemas/common-shapes.js'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; import { InvokeScreen } from '../../tui/screens/invoke'; import { parseHeaderFlags } from '../shared/header-utils'; import { handleInvoke, loadInvokeConfig } from './action'; import { resolvePrompt } from './resolve-prompt'; -import type { InvokeOptions } from './types'; +import type { InvokeOptions, InvokeResult } from './types'; import { validateInvokeOptions } from './validate'; import type { Command } from '@commander-js/extra-typings'; import { Text, render } from 'ink'; @@ -28,15 +30,16 @@ function stopSpinner(spinner: NodeJS.Timeout): void { process.stderr.write('\r\x1b[K'); // Clear line } -async function handleInvokeCLI(options: InvokeOptions): Promise { +function resolveProtocol(options: InvokeOptions, projectProtocol?: string): string { + if (projectProtocol) return projectProtocol.toLowerCase(); + if (options.tool) return 'mcp'; + return 'http'; +} + +async function handleInvokeCLI(options: InvokeOptions): Promise { const validation = validateInvokeOptions(options); if (!validation.valid) { - if (options.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); - } else { - console.error(validation.error); - } - process.exit(1); + return { success: false, error: new Error(validation.error) }; } let spinner: NodeJS.Timeout | undefined; @@ -55,44 +58,41 @@ async function handleInvokeCLI(options: InvokeOptions): Promise { stopSpinner(spinner); } - if (options.json) { - console.log(JSON.stringify(serializeResult(result))); - } else if (options.stream) { - // Streaming already wrote to stdout, just show session and log path - if (result.sessionId) { - console.error(`\nSession: ${result.sessionId}`); - console.error(`To resume: agentcore invoke --session-id ${result.sessionId}`); - } - if (result.logFilePath) { - console.error(`Log: ${result.logFilePath}`); - } - } else { - // Non-streaming, non-json: print provider info and response or error - if (result.success && result.response) { - console.log(result.response); - } else if (!result.success && result.error) { - console.error(result.error.message); - } - if (result.sessionId) { - console.error(`\nSession: ${result.sessionId}`); - console.error(`To resume: agentcore invoke --session-id ${result.sessionId}`); - } - if (result.logFilePath) { - console.error(`Log: ${result.logFilePath}`); - } - } - - process.exit(result.success ? 0 : 1); + return result; } catch (err) { if (spinner) { stopSpinner(spinner); } - if (options.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(err) })); - } else { - console.error(getErrorMessage(err)); + throw err; + } +} + +function printInvokeResult(result: InvokeResult, options: InvokeOptions): void { + if (options.json) { + console.log(JSON.stringify(serializeResult(result))); + } else if (options.stream) { + // Streaming already wrote to stdout, just show session and log path + if (result.sessionId) { + console.error(`\nSession: ${result.sessionId}`); + console.error(`To resume: agentcore invoke --session-id ${result.sessionId}`); + } + if (result.logFilePath) { + console.error(`Log: ${result.logFilePath}`); + } + } else { + // Non-streaming, non-json: print provider info and response or error + if (result.success && result.response) { + console.log(result.response); + } else if (!result.success && result.error) { + console.error(result.error.message); + } + if (result.sessionId) { + console.error(`\nSession: ${result.sessionId}`); + console.error(`To resume: agentcore invoke --session-id ${result.sessionId}`); + } + if (result.logFilePath) { + console.error(`Log: ${result.logFilePath}`); } - process.exit(1); } } @@ -149,6 +149,25 @@ export const registerInvoke = (program: Command) => { ) => { try { requireProject(); + + // Parse custom headers + let headers: Record | undefined; + if (cliOptions.header && cliOptions.header.length > 0) { + headers = parseHeaderFlags(cliOptions.header); + } + + // Determine protocol from project config (best-effort for telemetry) + let agentProtocol: string | undefined; + try { + const { project } = await loadInvokeConfig(); + const agent = cliOptions.runtime + ? project.runtimes.find(a => a.name === cliOptions.runtime) + : project.runtimes[0]; + agentProtocol = agent?.protocol; + } catch { + // Config load failure will be caught again inside handleInvokeCLI + } + // Resolve prompt from flag / positional / --prompt-file / stdin const resolved = await resolvePrompt({ flag: cliOptions.prompt, @@ -156,25 +175,12 @@ export const registerInvoke = (program: Command) => { file: cliOptions.promptFile, stdinPiped: !process.stdin.isTTY, }); - if (!resolved.success) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: resolved.error })); - } else { - console.error(resolved.error); - } - process.exit(1); - } - const prompt = resolved.prompt; - // Parse custom headers - let headers: Record | undefined; - if (cliOptions.header && cliOptions.header.length > 0) { - headers = parseHeaderFlags(cliOptions.header); - } - - // CLI mode if any CLI-specific options provided (follows deploy command pattern) + // CLI mode if any CLI-specific options provided, prompt resolved, or prompt resolution failed + // (follows deploy command pattern) if ( - prompt !== undefined || + !resolved.success || + resolved.prompt !== undefined || cliOptions.json || cliOptions.target || cliOptions.stream || @@ -183,8 +189,8 @@ export const registerInvoke = (program: Command) => { cliOptions.exec || cliOptions.bearerToken ) { - await handleInvokeCLI({ - prompt, + const options: InvokeOptions = { + prompt: resolved.prompt, agentName: cliOptions.runtime, targetName: cliOptions.target ?? 'default', sessionId: cliOptions.sessionId, @@ -197,21 +203,57 @@ export const registerInvoke = (program: Command) => { timeout: cliOptions.timeout, headers, bearerToken: cliOptions.bearerToken, - }); + }; + + const result = await withCommandRunTelemetry( + 'invoke', + { + has_stream: options.stream ?? false, + has_session_id: !!options.sessionId, + auth_type: standardize(AuthType, options.bearerToken ? 'bearer_token' : 'sigv4'), + protocol: standardize(Protocol, resolveProtocol(options, agentProtocol)), + }, + async (): Promise => { + if (!resolved.success) { + return { success: false, error: new Error(resolved.error ?? 'Prompt resolution failed') }; + } + return handleInvokeCLI(options); + } + ); + + printInvokeResult(result, options); + process.exit(result.success ? 0 : 1); } else { // No CLI options - interactive TUI mode (headers still passed if provided) requireTTY(); - const { waitUntilExit, unmount } = render( - unmount()} - initialSessionId={cliOptions.sessionId} - initialUserId={cliOptions.userId} - initialHeaders={headers} - initialBearerToken={cliOptions.bearerToken} - /> + + const tuiResult = await withCommandRunTelemetry( + 'invoke', + { + has_stream: true, + has_session_id: !!cliOptions.sessionId, + auth_type: standardize(AuthType, cliOptions.bearerToken ? 'bearer_token' : 'sigv4'), + protocol: standardize(Protocol, resolveProtocol({}, agentProtocol)), + }, + async (): Promise => { + const { waitUntilExit, unmount } = render( + unmount()} + initialSessionId={cliOptions.sessionId} + initialUserId={cliOptions.userId} + initialHeaders={headers} + initialBearerToken={cliOptions.bearerToken} + /> + ); + await waitUntilExit(); + return { success: true }; + } ); - await waitUntilExit(); + if (!tuiResult.success) { + render(Error: {getErrorMessage(tuiResult.error)}); + process.exit(1); + } } } catch (error) { if (cliOptions.json) { From 0de0e68b6ba01a2569deceff51d8d740ff91fa65 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 13 May 2026 18:15:47 +0000 Subject: [PATCH 2/2] fix: use ValidationError, eliminate double config load, move header parsing inside telemetry wrapper - Use ValidationError for validation and prompt resolution failures - Pass pre-loaded InvokeContext into handleInvokeCLI to avoid reading config files twice on the happy path - Move parseHeaderFlags inside the telemetry wrapper so invalid -H values are recorded as failures --- src/cli/commands/invoke/command.tsx | 90 +++++++++++++++++------------ 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/src/cli/commands/invoke/command.tsx b/src/cli/commands/invoke/command.tsx index 2ec2e5b10..0f2be6710 100644 --- a/src/cli/commands/invoke/command.tsx +++ b/src/cli/commands/invoke/command.tsx @@ -1,4 +1,4 @@ -import { type Result, serializeResult } from '../../../lib'; +import { type Result, ValidationError, serializeResult } from '../../../lib'; import { getErrorMessage } from '../../errors'; import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { AuthType, Protocol, standardize } from '../../telemetry/schemas/common-shapes.js'; @@ -6,7 +6,7 @@ import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; import { InvokeScreen } from '../../tui/screens/invoke'; import { parseHeaderFlags } from '../shared/header-utils'; -import { handleInvoke, loadInvokeConfig } from './action'; +import { type InvokeContext, handleInvoke, loadInvokeConfig } from './action'; import { resolvePrompt } from './resolve-prompt'; import type { InvokeOptions, InvokeResult } from './types'; import { validateInvokeOptions } from './validate'; @@ -36,16 +36,16 @@ function resolveProtocol(options: InvokeOptions, projectProtocol?: string): stri return 'http'; } -async function handleInvokeCLI(options: InvokeOptions): Promise { +async function handleInvokeCLI(options: InvokeOptions, preloadedContext?: InvokeContext): Promise { const validation = validateInvokeOptions(options); if (!validation.valid) { - return { success: false, error: new Error(validation.error) }; + return { success: false, error: new ValidationError(validation.error ?? 'Validation failed') }; } let spinner: NodeJS.Timeout | undefined; try { - const context = await loadInvokeConfig(); + const context = preloadedContext ?? (await loadInvokeConfig()); // Show spinner for non-streaming, non-json, non-exec invocations if (!options.stream && !options.json && !options.exec) { @@ -150,19 +150,14 @@ export const registerInvoke = (program: Command) => { try { requireProject(); - // Parse custom headers - let headers: Record | undefined; - if (cliOptions.header && cliOptions.header.length > 0) { - headers = parseHeaderFlags(cliOptions.header); - } - - // Determine protocol from project config (best-effort for telemetry) + // Load config once for protocol resolution and to pass into handleInvokeCLI + let invokeContext: InvokeContext | undefined; let agentProtocol: string | undefined; try { - const { project } = await loadInvokeConfig(); + invokeContext = await loadInvokeConfig(); const agent = cliOptions.runtime - ? project.runtimes.find(a => a.name === cliOptions.runtime) - : project.runtimes[0]; + ? invokeContext.project.runtimes.find(a => a.name === cliOptions.runtime) + : invokeContext.project.runtimes[0]; agentProtocol = agent?.protocol; } catch { // Config load failure will be caught again inside handleInvokeCLI @@ -189,44 +184,63 @@ export const registerInvoke = (program: Command) => { cliOptions.exec || cliOptions.bearerToken ) { - const options: InvokeOptions = { - prompt: resolved.prompt, - agentName: cliOptions.runtime, - targetName: cliOptions.target ?? 'default', - sessionId: cliOptions.sessionId, - userId: cliOptions.userId, - json: cliOptions.json, - stream: cliOptions.stream, - tool: cliOptions.tool, - input: cliOptions.input, - exec: cliOptions.exec, - timeout: cliOptions.timeout, - headers, - bearerToken: cliOptions.bearerToken, - }; - const result = await withCommandRunTelemetry( 'invoke', { - has_stream: options.stream ?? false, - has_session_id: !!options.sessionId, - auth_type: standardize(AuthType, options.bearerToken ? 'bearer_token' : 'sigv4'), - protocol: standardize(Protocol, resolveProtocol(options, agentProtocol)), + has_stream: cliOptions.stream ?? false, + has_session_id: !!cliOptions.sessionId, + auth_type: standardize(AuthType, cliOptions.bearerToken ? 'bearer_token' : 'sigv4'), + protocol: standardize( + Protocol, + resolveProtocol({ tool: cliOptions.tool } as InvokeOptions, agentProtocol) + ), }, async (): Promise => { if (!resolved.success) { - return { success: false, error: new Error(resolved.error ?? 'Prompt resolution failed') }; + return { success: false, error: new ValidationError(resolved.error ?? 'Prompt resolution failed') }; } - return handleInvokeCLI(options); + + // Parse custom headers + let headers: Record | undefined; + if (cliOptions.header && cliOptions.header.length > 0) { + headers = parseHeaderFlags(cliOptions.header); + } + + const options: InvokeOptions = { + prompt: resolved.prompt, + agentName: cliOptions.runtime, + targetName: cliOptions.target ?? 'default', + sessionId: cliOptions.sessionId, + userId: cliOptions.userId, + json: cliOptions.json, + stream: cliOptions.stream, + tool: cliOptions.tool, + input: cliOptions.input, + exec: cliOptions.exec, + timeout: cliOptions.timeout, + headers, + bearerToken: cliOptions.bearerToken, + }; + + return handleInvokeCLI(options, invokeContext); } ); - printInvokeResult(result, options); + printInvokeResult(result, { + json: cliOptions.json, + stream: cliOptions.stream, + }); process.exit(result.success ? 0 : 1); } else { // No CLI options - interactive TUI mode (headers still passed if provided) requireTTY(); + // Parse custom headers for TUI mode + let headers: Record | undefined; + if (cliOptions.header && cliOptions.header.length > 0) { + headers = parseHeaderFlags(cliOptions.header); + } + const tuiResult = await withCommandRunTelemetry( 'invoke', {