From e624baf8257aa20b6e32ff1cb55bf650fb27525c Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 2 Jan 2026 14:44:12 -0500 Subject: [PATCH 1/8] Add agent context system with secure token storage for Addie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add agent_contexts and agent_test_history tables for persisting agent URLs - Implement AES-256-GCM encrypted token storage with org-specific keys - Add save_agent, list_saved_agents, remove_saved_agent tools to Addie - Extend test_adcp_agent to auto-lookup saved credentials and record history - Add agent-tester.ts with E2E test scenarios including response_consistency 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- server/src/addie/mcp/agent-tester.ts | 2570 +++++++++++++++++ server/src/addie/mcp/member-tools.ts | 392 +++ server/src/db/agent-context-db.ts | 534 ++++ .../src/db/migrations/079_agent_contexts.sql | 136 + 4 files changed, 3632 insertions(+) create mode 100644 server/src/addie/mcp/agent-tester.ts create mode 100644 server/src/db/agent-context-db.ts create mode 100644 server/src/db/migrations/079_agent_contexts.sql diff --git a/server/src/addie/mcp/agent-tester.ts b/server/src/addie/mcp/agent-tester.ts new file mode 100644 index 0000000000..2f77b307b4 --- /dev/null +++ b/server/src/addie/mcp/agent-tester.ts @@ -0,0 +1,2570 @@ +/** + * AdCP Agent E2E Tester + * + * Provides comprehensive end-to-end testing of AdCP agents (sales, creative, signals). + * Replaces the testing.adcontextprotocol.org harness with conversational testing via Addie. + * + * Features: + * - Channel-aware testing (only tests features the agent supports) + * - Optional dry-run mode (real testing requires actual media buys) + * - Comprehensive scenario coverage based on AdCP spec + * - Schema validation via @adcp/client + */ + +import { AdCPClient } from '@adcp/client'; +import { logger } from '../../logger.js'; + +// Generic task result from executeTask - we use any for data since responses vary by task +interface TaskResult { + success: boolean; + data?: any; + error?: string; +} + +// Test scenarios that can be run +export type TestScenario = + | 'health_check' // Just check if agent responds + | 'discovery' // get_products, list_creative_formats, list_authorized_properties + | 'create_media_buy' // Discovery + create a test media buy + | 'full_sales_flow' // Full lifecycle: discovery -> create -> update -> delivery + | 'creative_sync' // Test sync_creatives flow + | 'creative_inline' // Test inline creatives in create_media_buy + | 'creative_reference' // Test reference creatives (creative_ids) + | 'pricing_models' // Test different pricing models the agent supports + | 'creative_flow' // Creative agent: list_formats -> build -> preview + | 'signals_flow' // Signals agent: get_signals -> activate + // Edge case testing scenarios + | 'error_handling' // Test agent returns proper error responses + | 'validation' // Test schema validation (invalid inputs should be rejected) + | 'pricing_edge_cases' // Test auction vs fixed pricing, min spend, bid_price requirements + | 'temporal_validation' // Test date/time ordering and format validation + // Behavioral analysis scenarios + | 'behavior_analysis' // Analyze agent behavior: auth requirements, brief relevance, filtering + // Response consistency scenarios + | 'response_consistency'; // Check for schema errors, pagination bugs, data mismatches + +export interface TestOptions { + // Custom brief for product discovery + brief?: string; + // Budget for test media buy (default: 1000) + budget?: number; + // Specific format IDs to test + format_ids?: string[]; + // Test session ID for isolation + test_session_id?: string; + // Whether to use dry-run mode (default: true for safety) + dry_run?: boolean; + // Channels to focus on (if not specified, tests all agent supports) + channels?: string[]; + // Specific pricing models to test + pricing_models?: string[]; + // Authentication for agents that require it + auth?: { + type: 'bearer'; + token: string; + }; +} + +export interface TestStepResult { + step: string; + task?: string; + passed: boolean; + duration_ms: number; + details?: string; + error?: string; + response_preview?: string; + // For tracking what was created (for cleanup or follow-up) + created_id?: string; +} + +export interface AgentProfile { + name: string; + tools: string[]; + channels?: string[]; + pricing_models?: string[]; + format_ids?: string[]; + delivery_types?: string[]; +} + +export interface TestResult { + agent_url: string; + scenario: TestScenario; + overall_passed: boolean; + steps: TestStepResult[]; + summary: string; + total_duration_ms: number; + tested_at: string; + // Agent profile discovered during testing + agent_profile?: AgentProfile; + // Was this run in dry-run mode? + dry_run: boolean; +} + +/** + * Create a test client for an agent + */ +function createTestClient( + agentUrl: string, + protocol: 'mcp' | 'a2a' = 'mcp', + options: TestOptions = {} +) { + const headers: Record = {}; + + // Dry-run is true by default for safety + if (options.dry_run !== false) { + headers['X-Dry-Run'] = 'true'; + } + + if (options.test_session_id) { + headers['X-Test-Session-ID'] = options.test_session_id; + } + + // Build agent config with auth_token if provided + const agentConfig: { + id: string; + name: string; + agent_uri: string; + protocol: 'mcp' | 'a2a'; + auth_token?: string; + } = { + id: 'test', + name: 'E2E Test Client', + agent_uri: agentUrl, + protocol, + }; + + // Add auth_token to agent config - the library will use it automatically + if (options.auth?.type === 'bearer' && options.auth?.token) { + agentConfig.auth_token = options.auth.token; + } + + const multiClient = new AdCPClient([agentConfig], { + headers, + }); + + return multiClient.agent('test'); +} + +/** + * Run a single test step with timing + */ +async function runStep( + stepName: string, + taskName: string | undefined, + fn: () => Promise +): Promise<{ result?: T; step: TestStepResult }> { + const start = Date.now(); + try { + const result = await fn(); + const duration = Date.now() - start; + return { + result, + step: { + step: stepName, + task: taskName, + passed: true, + duration_ms: duration, + }, + }; + } catch (error) { + const duration = Date.now() - start; + const errorMessage = error instanceof Error ? error.message : String(error); + return { + step: { + step: stepName, + task: taskName, + passed: false, + duration_ms: duration, + error: errorMessage, + }, + }; + } +} + +/** + * Discover agent profile - what capabilities does this agent have? + */ +async function discoverAgentProfile( + client: ReturnType +): Promise<{ profile: AgentProfile; step: TestStepResult }> { + const { result: agentInfo, step } = await runStep( + 'Discover agent capabilities', + 'getAgentInfo', + () => client.getAgentInfo() + ); + + const profile: AgentProfile = { + name: agentInfo?.name || 'Unknown', + tools: agentInfo?.tools?.map((t: { name: string }) => t.name) || [], + }; + + if (agentInfo) { + step.details = `Agent: ${profile.name}, Tools: ${profile.tools.length}`; + step.response_preview = JSON.stringify({ + name: profile.name, + tools: profile.tools, + }, null, 2); + } + + return { profile, step }; +} + +/** + * Discover what channels, pricing models, formats the agent supports + * by calling get_products and analyzing the response + */ +async function discoverAgentCapabilities( + client: ReturnType, + profile: AgentProfile, + options: TestOptions +): Promise<{ capabilities: Partial; steps: TestStepResult[] }> { + const steps: TestStepResult[] = []; + const capabilities: Partial = {}; + + if (!profile.tools.includes('get_products')) { + return { capabilities, steps }; + } + + const brief = options.brief || 'Show me all available advertising products across all channels'; + // Include brand_manifest as some agents require it (e.g., tenant-specific agents) + const getProductsParams: Record = { + brief, + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + }, + }; + const { result, step } = await runStep( + 'Discover products for capability analysis', + 'get_products', + async () => client.executeTask('get_products', getProductsParams) as Promise + ); + + if (result?.success && result?.data?.products) { + const products = result.data.products as any[]; + + // Extract unique channels + const channels = new Set(); + const pricingModels = new Set(); + const formatIds = new Set(); + const deliveryTypes = new Set(); + + for (const product of products) { + // Channels from product + if (product.channels) { + for (const ch of product.channels) { + channels.add(ch); + } + } + // Delivery type + if (product.delivery_type) { + deliveryTypes.add(product.delivery_type); + } + // Pricing models + if (product.pricing_options) { + for (const po of product.pricing_options) { + if (po.model) pricingModels.add(po.model); + } + } + // Format IDs + if (product.format_ids) { + for (const fid of product.format_ids) { + const id = typeof fid === 'string' ? fid : fid.id; + if (id) formatIds.add(id); + } + } + } + + capabilities.channels = Array.from(channels); + capabilities.pricing_models = Array.from(pricingModels); + capabilities.format_ids = Array.from(formatIds); + capabilities.delivery_types = Array.from(deliveryTypes); + + step.details = `Found ${products.length} products across ${channels.size} channel(s), ${pricingModels.size} pricing model(s)`; + step.response_preview = JSON.stringify({ + products_count: products.length, + channels: capabilities.channels, + pricing_models: capabilities.pricing_models, + delivery_types: capabilities.delivery_types, + format_count: capabilities.format_ids?.length, + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'get_products failed'; + } + + steps.push(step); + return { capabilities, steps }; +} + +/** + * Test: Health Check + * Verifies the agent is responding and has an agent card + */ +async function testHealthCheck(agentUrl: string, options: TestOptions): Promise { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { step } = await discoverAgentProfile(client); + steps.push(step); + + return steps; +} + +/** + * Test: Discovery + * Tests product discovery, format listing, and property listing + */ +async function testDiscovery(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // Discover agent profile + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // Discover capabilities + const { capabilities, steps: capSteps } = await discoverAgentCapabilities(client, profile, options); + steps.push(...capSteps); + + // Merge capabilities into profile + Object.assign(profile, capabilities); + + // List creative formats (if available) + if (profile.tools.includes('list_creative_formats')) { + const { result, step } = await runStep( + 'List creative formats', + 'list_creative_formats', + async () => client.executeTask('list_creative_formats', {}) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const formatCount = data.format_ids?.length || data.formats?.length || 0; + const creativeAgents = data.creative_agents || []; + step.details = `Found ${formatCount} format(s), ${creativeAgents.length} creative agent(s)`; + step.response_preview = JSON.stringify({ + format_ids: (data.format_ids || data.formats?.map((f: any) => f.format_id))?.slice(0, 5), + creative_agents: creativeAgents.map((a: any) => a.agent_url || a.url), + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'list_creative_formats returned unsuccessful result'; + } + steps.push(step); + } + + // List authorized properties (if available) + if (profile.tools.includes('list_authorized_properties')) { + const { result, step } = await runStep( + 'List authorized properties', + 'list_authorized_properties', + async () => client.executeTask('list_authorized_properties', {}) as Promise + ); + + const properties = result?.data?.authorized_properties as any[] | undefined; + if (result?.success && properties) { + step.details = `Found ${properties.length} authorized propert(ies)`; + step.response_preview = JSON.stringify({ + properties_count: properties.length, + domains: properties.slice(0, 3).map((p: any) => p.domain), + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'list_authorized_properties returned unsuccessful result'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Find a suitable product for testing based on options + */ +function selectProduct(products: any[], options: TestOptions): any | null { + // If channels specified, filter to matching products + let candidates = products; + + if (options.channels?.length) { + candidates = products.filter(p => + p.channels?.some((ch: string) => options.channels!.includes(ch)) + ); + } + + // If pricing models specified, filter further + if (options.pricing_models?.length) { + candidates = candidates.filter(p => + p.pricing_options?.some((po: any) => + options.pricing_models!.includes(po.model) + ) + ); + } + + // Return first matching or first product + return candidates[0] || products[0] || null; +} + +/** + * Select a pricing option from a product + */ +function selectPricingOption(product: any, preferredModels?: string[]): any | null { + const options = product.pricing_options || []; + + if (preferredModels?.length) { + const preferred = options.find((po: any) => preferredModels.includes(po.model)); + if (preferred) return preferred; + } + + return options[0] || null; +} + +/** + * Build a create_media_buy request + */ +function buildCreateMediaBuyRequest( + product: any, + pricingOption: any, + options: TestOptions, + extras: { + inline_creatives?: any[]; + creative_ids?: string[]; + } = {} +): any { + const budget = options.budget || 1000; + const now = new Date(); + const startTime = new Date(now.getTime() + 24 * 60 * 60 * 1000); // Tomorrow + const endTime = new Date(startTime.getTime() + 7 * 24 * 60 * 60 * 1000); // 7 days later + + const isAuction = pricingOption.model === 'auction' || + pricingOption.is_fixed === false || + pricingOption.floor_price !== undefined; + + const packageRequest: any = { + buyer_ref: `pkg-test-${Date.now()}`, + product_id: product.product_id, + budget, + pricing_option_id: pricingOption.pricing_option_id, + }; + + // Add bid_price if auction-based + if (isAuction && pricingOption.floor_price) { + packageRequest.bid_price = pricingOption.floor_price * 1.5; + } + + // Add inline creatives if provided + if (extras.inline_creatives?.length) { + packageRequest.creatives = extras.inline_creatives; + } + + // Add creative references if provided + if (extras.creative_ids?.length) { + packageRequest.creative_ids = extras.creative_ids; + } + + return { + buyer_ref: `e2e-test-${Date.now()}`, + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + }, + start_time: startTime.toISOString(), + end_time: endTime.toISOString(), + packages: [packageRequest], + }; +} + +/** + * Test: Create Media Buy + * Discovers products, then creates a test media buy + */ +async function testCreateMediaBuy(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile; mediaBuyId?: string }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // First run discovery + const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); + steps.push(...discoverySteps); + + if (!profile?.tools.includes('create_media_buy')) { + steps.push({ + step: 'Create media buy', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: 'Agent does not support create_media_buy', + }); + return { steps, profile }; + } + + // Get products + const { result: productsResult } = await runStep( + 'Fetch products for media buy', + 'get_products', + async () => client.executeTask('get_products', { + brief: options.brief || 'Looking for display advertising products', + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + }, + }) as Promise + ); + + const products = productsResult?.data?.products as any[] | undefined; + if (!productsResult?.success || !products?.length) { + steps.push({ + step: 'Create media buy', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: 'No products available to create media buy', + }); + return { steps, profile }; + } + + const product = selectProduct(products, options); + const pricingOption = selectPricingOption(product, options.pricing_models); + + if (!pricingOption) { + steps.push({ + step: 'Create media buy', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: `Product "${product.name}" has no pricing options`, + }); + return { steps, profile }; + } + + const createRequest = buildCreateMediaBuyRequest(product, pricingOption, options); + + // Create the media buy + const { result: createResult, step: createStep } = await runStep( + 'Create media buy', + 'create_media_buy', + async () => client.executeTask('create_media_buy', createRequest) as Promise + ); + + let mediaBuyId: string | undefined; + + if (createResult?.success && createResult?.data) { + const mediaBuy = createResult.data as any; + mediaBuyId = mediaBuy.media_buy_id || mediaBuy.media_buy?.media_buy_id; + const status = mediaBuy.status || mediaBuy.media_buy?.status; + const packages = mediaBuy.packages || mediaBuy.media_buy?.packages; + createStep.details = `Created media buy: ${mediaBuyId}, status: ${status}`; + createStep.created_id = mediaBuyId; + createStep.response_preview = JSON.stringify({ + media_buy_id: mediaBuyId, + status, + packages_count: packages?.length, + pricing_model: pricingOption.model, + product_name: product.name, + }, null, 2); + } else if (createResult && !createResult.success) { + createStep.passed = false; + createStep.error = createResult.error || 'create_media_buy returned unsuccessful result'; + } + steps.push(createStep); + + return { steps, profile, mediaBuyId }; +} + +/** + * Test: Full Sales Flow + * Complete lifecycle: discovery -> create -> update -> delivery + */ +async function testFullSalesFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // Run create media buy flow first + const { steps: createSteps, profile, mediaBuyId } = await testCreateMediaBuy(agentUrl, options); + steps.push(...createSteps); + + if (!mediaBuyId) { + return { steps, profile }; + } + + // Test update_media_buy if available + if (profile?.tools.includes('update_media_buy')) { + const { result: updateResult, step: updateStep } = await runStep( + 'Update media buy (increase budget)', + 'update_media_buy', + async () => client.executeTask('update_media_buy', { + media_buy_id: mediaBuyId, + packages: [{ + package_id: 'pkg-0', + budget: (options.budget || 1000) * 1.5, + }], + }) as Promise + ); + + if (updateResult?.success && updateResult?.data) { + const data = updateResult.data as any; + const status = data.status || data.media_buy?.status; + updateStep.details = `Updated media buy, status: ${status}`; + updateStep.response_preview = JSON.stringify({ + media_buy_id: data.media_buy_id || data.media_buy?.media_buy_id, + status, + }, null, 2); + } else if (updateResult && !updateResult.success) { + updateStep.passed = false; + updateStep.error = updateResult.error || 'update_media_buy returned unsuccessful result'; + } + steps.push(updateStep); + } + + // Test get_media_buy_delivery if available + if (profile?.tools.includes('get_media_buy_delivery')) { + const { result: deliveryResult, step: deliveryStep } = await runStep( + 'Get delivery metrics', + 'get_media_buy_delivery', + async () => client.executeTask('get_media_buy_delivery', { + media_buy_ids: [mediaBuyId], + }) as Promise + ); + + if (deliveryResult?.success && deliveryResult?.data) { + const delivery = deliveryResult.data as any; + deliveryStep.details = `Retrieved delivery metrics`; + deliveryStep.response_preview = JSON.stringify({ + has_deliveries: !!(delivery.deliveries?.length || delivery.media_buys?.length), + }, null, 2); + } else if (deliveryResult && !deliveryResult.success) { + deliveryStep.passed = false; + deliveryStep.error = deliveryResult.error || 'get_media_buy_delivery returned unsuccessful result'; + } + steps.push(deliveryStep); + } + + return { steps, profile }; +} + +/** + * Test: Creative Sync Flow + * Tests sync_creatives separately from create_media_buy + */ +async function testCreativeSync(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // Discover profile + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profile.tools.includes('sync_creatives')) { + steps.push({ + step: 'Sync creatives', + task: 'sync_creatives', + passed: false, + duration_ms: 0, + error: 'Agent does not support sync_creatives', + }); + return { steps, profile }; + } + + // Get format info first + let formatId = 'display_300x250'; // Default + if (profile.tools.includes('list_creative_formats')) { + const { result: formatsResult } = await runStep( + 'Get formats for creative', + 'list_creative_formats', + async () => client.executeTask('list_creative_formats', {}) as Promise + ); + + if (formatsResult?.success && formatsResult?.data) { + const data = formatsResult.data as any; + const firstFormat = data.format_ids?.[0] || data.formats?.[0]; + if (firstFormat) { + formatId = typeof firstFormat === 'string' ? firstFormat : (firstFormat.id || firstFormat.format_id); + } + } + } + + // Test sync_creatives with a simple creative + // Assets must be an object keyed by asset_role, not an array + const testCreative = { + creative_id: `test-creative-${Date.now()}`, + name: 'E2E Test Creative', + format_id: formatId, + assets: { + primary: { + url: 'https://via.placeholder.com/300x250', + width: 300, + height: 250, + format: 'png', + }, + }, + }; + + const { result: syncResult, step: syncStep } = await runStep( + 'Sync creative to library', + 'sync_creatives', + async () => client.executeTask('sync_creatives', { + creatives: [testCreative], + }) as Promise + ); + + if (syncResult?.success && syncResult?.data) { + const data = syncResult.data as any; + const creatives = data.creatives || []; + const actions = creatives.map((c: any) => c.action); + syncStep.details = `Synced ${creatives.length} creative(s), actions: ${actions.join(', ')}`; + syncStep.response_preview = JSON.stringify({ + creatives_count: creatives.length, + actions: actions, + creative_ids: creatives.map((c: any) => c.creative_id), + }, null, 2); + } else if (syncResult && !syncResult.success) { + syncStep.passed = false; + syncStep.error = syncResult.error || 'sync_creatives returned unsuccessful result'; + } + steps.push(syncStep); + + // Test list_creatives if available + if (profile.tools.includes('list_creatives')) { + const { result: listResult, step: listStep } = await runStep( + 'List creatives in library', + 'list_creatives', + async () => client.executeTask('list_creatives', {}) as Promise + ); + + if (listResult?.success && listResult?.data) { + const data = listResult.data as any; + const creatives = data.creatives || []; + const querySummary = data.query_summary; + const totalMatching = querySummary?.total_matching; + const returned = querySummary?.returned ?? creatives.length; + + // Check for pagination bug: total_matching > 0 but returned = 0 + if (totalMatching !== undefined && totalMatching > 0 && returned === 0) { + listStep.passed = false; + listStep.error = `Pagination bug: query_summary shows ${totalMatching} total_matching but returned ${returned} creatives`; + listStep.response_preview = JSON.stringify({ + total_matching: totalMatching, + returned, + creatives_count: creatives.length, + pagination: data.pagination, + }, null, 2); + } else { + listStep.details = `Found ${creatives.length} creative(s) in library`; + listStep.response_preview = JSON.stringify({ + creatives_count: creatives.length, + total_matching: totalMatching, + statuses: Array.from(new Set(creatives.map((c: any) => c.status))), + }, null, 2); + } + } else if (listResult && !listResult.success) { + listStep.passed = false; + listStep.error = listResult.error || 'list_creatives returned unsuccessful result'; + } + steps.push(listStep); + } + + return { steps, profile }; +} + +/** + * Test: Creative Inline Flow + * Tests providing creatives inline in create_media_buy + */ +async function testCreativeInline(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // Discovery first + const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); + steps.push(...discoverySteps); + + if (!profile?.tools.includes('create_media_buy')) { + steps.push({ + step: 'Create media buy with inline creatives', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: 'Agent does not support create_media_buy', + }); + return { steps, profile }; + } + + // Get products + const { result: productsResult } = await runStep( + 'Fetch products', + 'get_products', + async () => client.executeTask('get_products', { + brief: options.brief || 'Looking for display advertising products', + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + }, + }) as Promise + ); + + const products = productsResult?.data?.products as any[] | undefined; + if (!products?.length) { + steps.push({ + step: 'Create media buy with inline creatives', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: 'No products available', + }); + return { steps, profile }; + } + + const product = selectProduct(products, options); + const pricingOption = selectPricingOption(product, options.pricing_models); + + if (!pricingOption) { + steps.push({ + step: 'Create media buy with inline creatives', + task: 'create_media_buy', + passed: false, + duration_ms: 0, + error: 'No pricing options available', + }); + return { steps, profile }; + } + + // Build inline creative + const formatId = product.format_ids?.[0]; + const formatIdValue = typeof formatId === 'string' ? formatId : (formatId?.id || 'display_300x250'); + + // Assets must be an object keyed by asset_role, not an array + const inlineCreative = { + creative_id: `inline-creative-${Date.now()}`, + name: 'Inline Test Creative', + format_id: formatIdValue, + assets: { + primary: { + url: 'https://via.placeholder.com/300x250', + width: 300, + height: 250, + format: 'png', + }, + }, + }; + + const createRequest = buildCreateMediaBuyRequest(product, pricingOption, options, { + inline_creatives: [inlineCreative], + }); + + const { result: createResult, step: createStep } = await runStep( + 'Create media buy with inline creatives', + 'create_media_buy', + async () => client.executeTask('create_media_buy', createRequest) as Promise + ); + + if (createResult?.success && createResult?.data) { + const mediaBuy = createResult.data as any; + const mediaBuyId = mediaBuy.media_buy_id || mediaBuy.media_buy?.media_buy_id; + createStep.details = `Created media buy with inline creative: ${mediaBuyId}`; + createStep.response_preview = JSON.stringify({ + media_buy_id: mediaBuyId, + status: mediaBuy.status || mediaBuy.media_buy?.status, + inline_creative_used: true, + }, null, 2); + } else if (createResult && !createResult.success) { + createStep.passed = false; + createStep.error = createResult.error || 'create_media_buy returned unsuccessful result'; + } + steps.push(createStep); + + return { steps, profile }; +} + +/** + * Test: Pricing Models + * Tests different pricing models the agent supports + */ +async function testPricingModels(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // Discovery first + const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); + steps.push(...discoverySteps); + + if (!profile?.pricing_models?.length) { + steps.push({ + step: 'Test pricing models', + passed: false, + duration_ms: 0, + error: 'No pricing models discovered', + }); + return { steps, profile }; + } + + // Get products to analyze pricing + const { result: productsResult } = await runStep( + 'Fetch products for pricing analysis', + 'get_products', + async () => client.executeTask('get_products', { + brief: 'Show all products', + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + }, + }) as Promise + ); + + const products = productsResult?.data?.products as any[] | undefined; + if (!products?.length) { + return { steps, profile }; + } + + // Analyze pricing model distribution + const pricingAnalysis: Record = {}; + + for (const product of products) { + for (const po of (product.pricing_options || [])) { + const model = po.model || 'unknown'; + if (!pricingAnalysis[model]) { + pricingAnalysis[model] = { count: 0, fixed: 0, auction: 0 }; + } + pricingAnalysis[model].count++; + if (po.is_fixed === false || po.floor_price !== undefined) { + pricingAnalysis[model].auction++; + } else { + pricingAnalysis[model].fixed++; + } + } + } + + steps.push({ + step: 'Analyze pricing models', + passed: true, + duration_ms: 0, + details: `Found ${Object.keys(pricingAnalysis).length} pricing model(s)`, + response_preview: JSON.stringify(pricingAnalysis, null, 2), + }); + + return { steps, profile }; +} + +/** + * Test: Creative Flow (for creative agents) + */ +async function testCreativeFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // List creative formats + if (profile.tools.includes('list_creative_formats')) { + const { result, step } = await runStep( + 'List creative formats', + 'list_creative_formats', + async () => client.executeTask('list_creative_formats', {}) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const formats = data.formats || []; + step.details = `Found ${formats.length} format(s)`; + step.response_preview = JSON.stringify({ + formats_count: formats.length, + format_names: formats.slice(0, 5).map((f: any) => f.name || f.format_id), + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'list_creative_formats failed'; + } + steps.push(step); + } + + // Build creative (if available) + if (profile.tools.includes('build_creative')) { + const { result, step } = await runStep( + 'Build creative', + 'build_creative', + async () => client.executeTask('build_creative', { + format_id: options.format_ids?.[0] || 'display_300x250', + brand_manifest: { + name: 'E2E Test Brand', + url: 'https://test.example.com', + tagline: 'Testing the future of advertising', + }, + prompt: 'Create a simple display ad for a tech product', + }) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + step.details = `Built creative successfully`; + step.response_preview = JSON.stringify({ + creative_id: data.creative_id || data.creative?.creative_id, + format_id: data.format_id || data.creative?.format_id, + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'build_creative failed'; + } + steps.push(step); + } + + // Preview creative (if available) + if (profile.tools.includes('preview_creative')) { + const { result, step } = await runStep( + 'Preview creative', + 'preview_creative', + async () => client.executeTask('preview_creative', { + creative: { + format_id: options.format_ids?.[0] || 'display_300x250', + name: 'Test Creative', + assets: [], + }, + }) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + step.details = `Generated preview`; + step.response_preview = JSON.stringify({ + has_renders: !!(data.renders?.length || data.preview_url), + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'preview_creative failed'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Test: Signals Flow (for signals agents) + */ +async function testSignalsFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // Get signals + if (profile.tools.includes('get_signals')) { + const { result, step } = await runStep( + 'Get signals', + 'get_signals', + async () => client.executeTask('get_signals', { + brief: 'Looking for audience segments interested in technology', + }) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const signals = data.signals || []; + step.details = `Found ${signals.length} signal(s)`; + step.response_preview = JSON.stringify({ + signals_count: signals.length, + signal_names: signals.slice(0, 3).map((s: any) => s.name), + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'get_signals failed'; + } + steps.push(step); + } + + // Activate signal (if available) + if (profile.tools.includes('activate_signal')) { + const { result, step } = await runStep( + 'Activate signal', + 'activate_signal', + async () => client.executeTask('activate_signal', { + signal_id: 'test-signal-id', + destination: { + platform: 'test-platform', + account_id: 'test-account', + }, + }) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + step.details = `Signal activation submitted`; + step.response_preview = JSON.stringify({ + status: data.status || data.deployment?.status, + }, null, 2); + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'activate_signal failed'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Test: Error Handling + * Verifies the agent returns proper discriminated union error responses + */ +async function testErrorHandling(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // Test 1: Invalid product_id in create_media_buy should return proper error + if (profile.tools.includes('create_media_buy')) { + const { result, step } = await runStep( + 'Invalid product_id error response', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `error-test-${Date.now()}`, + brand_manifest: { + name: 'Error Test Brand', + url: 'https://test.example.com', + }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-error-test', + product_id: 'NONEXISTENT_PRODUCT_ID_12345', + budget: 1000, + pricing_option_id: 'nonexistent-pricing', + }], + }) as Promise + ); + + // For error handling test, we EXPECT an error - passing means the error was returned properly + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent correctly returned error for invalid product_id'; + step.response_preview = JSON.stringify({ error: result.error }, null, 2); + } else if (result?.success) { + step.passed = false; + step.error = 'Agent accepted invalid product_id - should have returned error'; + } else { + step.passed = false; + step.error = 'Agent returned neither success nor proper error response'; + } + steps.push(step); + } + + // Test 2: Missing required field in get_products (brief is often required) + if (profile.tools.includes('get_products')) { + const { result, step } = await runStep( + 'Empty request handling', + 'get_products', + async () => client.executeTask('get_products', {}) as Promise + ); + + // Some agents may accept empty requests, others may require brief + if (result?.success) { + step.passed = true; + step.details = 'Agent accepts empty get_products request (permissive)'; + } else if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent requires brief/brand_manifest (stricter validation)'; + step.response_preview = JSON.stringify({ error: result.error }, null, 2); + } else { + step.passed = false; + step.error = 'Unclear response - neither success nor proper error'; + } + steps.push(step); + } + + // Test 3: Invalid format_id in sync_creatives + if (profile.tools.includes('sync_creatives')) { + const { result, step } = await runStep( + 'Invalid format_id error response', + 'sync_creatives', + async () => client.executeTask('sync_creatives', { + creatives: [{ + creative_id: `invalid-format-test-${Date.now()}`, + name: 'Invalid Format Test', + format_id: 'TOTALLY_INVALID_FORMAT_ID_999', + assets: { + primary: { + url: 'https://via.placeholder.com/300x250', + width: 300, + height: 250, + format: 'png', + }, + }, + }], + }) as Promise + ); + + // Expect error for invalid format_id + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent correctly rejected invalid format_id'; + step.response_preview = JSON.stringify({ error: result.error }, null, 2); + } else if (result?.success) { + // Some agents may be permissive and accept any format_id + step.passed = true; + step.details = 'Agent accepts unknown format_ids (permissive mode)'; + } else { + step.passed = false; + step.error = 'Unclear response for invalid format_id'; + } + steps.push(step); + } + + // Test 4: get_media_buy_delivery with non-existent media_buy_id + if (profile.tools.includes('get_media_buy_delivery')) { + const { result, step } = await runStep( + 'Non-existent media_buy_id error', + 'get_media_buy_delivery', + async () => client.executeTask('get_media_buy_delivery', { + media_buy_ids: ['NONEXISTENT_MEDIA_BUY_ID_99999'], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent correctly returned error for non-existent media buy'; + step.response_preview = JSON.stringify({ error: result.error }, null, 2); + } else if (result?.success) { + // Check if it returned empty deliveries (also acceptable) + const data = result.data as any; + const deliveries = data?.deliveries || data?.media_buys || []; + if (deliveries.length === 0) { + step.passed = true; + step.details = 'Agent returned empty deliveries for non-existent media buy'; + } else { + step.passed = false; + step.error = 'Agent returned deliveries for non-existent media_buy_id'; + } + } else { + step.passed = false; + step.error = 'Unclear response for non-existent media_buy_id'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Test: Validation + * Tests that agents properly validate inputs and reject malformed requests + */ +async function testValidation(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // Test 1: Invalid enum value for pacing + if (profile.tools.includes('create_media_buy')) { + const { result, step } = await runStep( + 'Invalid pacing enum value', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `validation-test-${Date.now()}`, + brand_manifest: { name: 'Validation Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-validation', + product_id: 'test-product', + budget: 1000, + pricing_option_id: 'test-pricing', + pacing: 'INVALID_PACING_VALUE' as any, // Invalid - should be even/asap/front_loaded + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent rejected invalid pacing enum value'; + } else if (result?.success) { + step.passed = false; + step.error = 'Agent accepted invalid pacing value - should validate enums'; + } else { + step.passed = false; + step.error = 'Unclear validation response'; + } + steps.push(step); + } + + // Test 2: Zero budget package + if (profile.tools.includes('create_media_buy')) { + const { result, step } = await runStep( + 'Zero budget validation', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `zero-budget-test-${Date.now()}`, + brand_manifest: { name: 'Zero Budget Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-zero-budget', + product_id: 'test-product', + budget: 0, // Zero budget - should be rejected or flagged + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent rejected zero budget (strict validation)'; + } else if (result?.success) { + step.passed = true; + step.details = 'Agent accepts zero budget (permissive - schema allows minimum: 0)'; + } else { + step.passed = false; + step.error = 'Unclear response for zero budget'; + } + steps.push(step); + } + + // Test 3: Negative budget (definitely invalid) + if (profile.tools.includes('create_media_buy')) { + const { result, step } = await runStep( + 'Negative budget rejection', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `negative-budget-test-${Date.now()}`, + brand_manifest: { name: 'Negative Budget Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-negative', + product_id: 'test-product', + budget: -500, // Negative budget - MUST be rejected + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent correctly rejected negative budget'; + } else if (result?.success) { + step.passed = false; + step.error = 'CRITICAL: Agent accepted negative budget - must validate minimum: 0'; + } else { + step.passed = false; + step.error = 'Unclear response for negative budget'; + } + steps.push(step); + } + + // Test 4: Invalid creative weight (> 100) + if (profile.tools.includes('sync_creatives')) { + const { result, step } = await runStep( + 'Invalid creative weight (> 100)', + 'sync_creatives', + async () => client.executeTask('sync_creatives', { + creatives: [{ + creative_id: `weight-test-${Date.now()}`, + name: 'Weight Test Creative', + format_id: 'display_300x250', + weight: 150, // Invalid - max is 100 + assets: { + primary: { + url: 'https://via.placeholder.com/300x250', + width: 300, + height: 250, + format: 'png', + }, + }, + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent rejected weight > 100'; + } else if (result?.success) { + step.passed = false; + step.error = 'Agent accepted weight > 100 - should validate maximum: 100'; + } else { + step.passed = false; + step.error = 'Unclear response for invalid weight'; + } + steps.push(step); + } + + // Test 5: Empty creatives array + if (profile.tools.includes('sync_creatives')) { + const { result, step } = await runStep( + 'Empty creatives array handling', + 'sync_creatives', + async () => client.executeTask('sync_creatives', { + creatives: [], // Empty array - should be rejected or return empty + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent rejected empty creatives array'; + } else if (result?.success) { + step.passed = true; + step.details = 'Agent accepts empty creatives array (returns empty result)'; + } else { + step.passed = false; + step.error = 'Unclear response for empty creatives array'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Test: Pricing Edge Cases + * Tests auction vs fixed pricing, min spend requirements, bid_price handling + */ +async function testPricingEdgeCases(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + // First get products to understand pricing options + const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); + steps.push(...discoverySteps); + + if (!profile?.tools.includes('create_media_buy') || !profile?.tools.includes('get_products')) { + steps.push({ + step: 'Pricing edge cases', + passed: false, + duration_ms: 0, + error: 'Agent does not support create_media_buy or get_products', + }); + return { steps, profile }; + } + + // Get products to find actual pricing options + const { result: productsResult } = await runStep( + 'Fetch products for pricing analysis', + 'get_products', + async () => client.executeTask('get_products', { + brief: 'Show all products with pricing details', + brand_manifest: { name: 'Pricing Test', url: 'https://test.example.com' }, + }) as Promise + ); + + const products = productsResult?.data?.products as any[] | undefined; + if (!products?.length) { + steps.push({ + step: 'Pricing edge cases', + passed: false, + duration_ms: 0, + error: 'No products available for pricing tests', + }); + return { steps, profile }; + } + + // Analyze products for auction vs fixed pricing + const auctionProducts: any[] = []; + const fixedProducts: any[] = []; + const productsWithMinSpend: any[] = []; + + for (const product of products) { + for (const po of (product.pricing_options || [])) { + if (po.is_fixed === false || po.floor_price !== undefined || po.price_guidance !== undefined) { + auctionProducts.push({ product, pricingOption: po }); + } else if (po.rate !== undefined) { + fixedProducts.push({ product, pricingOption: po }); + } + if (po.min_spend_per_package !== undefined && po.min_spend_per_package > 0) { + productsWithMinSpend.push({ product, pricingOption: po, minSpend: po.min_spend_per_package }); + } + } + } + + steps.push({ + step: 'Analyze pricing options', + passed: true, + duration_ms: 0, + details: `Found ${fixedProducts.length} fixed, ${auctionProducts.length} auction, ${productsWithMinSpend.length} with min spend`, + response_preview: JSON.stringify({ + fixed_count: fixedProducts.length, + auction_count: auctionProducts.length, + min_spend_count: productsWithMinSpend.length, + }, null, 2), + }); + + // Test 1: Auction pricing without bid_price (should fail) + if (auctionProducts.length > 0) { + const { product, pricingOption } = auctionProducts[0]; + const { result, step } = await runStep( + 'Auction pricing without bid_price', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `auction-no-bid-${Date.now()}`, + brand_manifest: { name: 'Auction Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-auction-no-bid', + product_id: product.product_id, + budget: 5000, + pricing_option_id: pricingOption.pricing_option_id, + // Intentionally missing bid_price for auction pricing + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent correctly requires bid_price for auction pricing'; + } else if (result?.success) { + step.passed = false; + step.error = 'Agent accepted auction pricing without bid_price - should require it'; + } else { + step.passed = false; + step.error = 'Unclear response for missing bid_price'; + } + steps.push(step); + } + + // Test 2: Fixed pricing with bid_price (should be ignored or rejected) + if (fixedProducts.length > 0) { + const { product, pricingOption } = fixedProducts[0]; + const { result, step } = await runStep( + 'Fixed pricing with unnecessary bid_price', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `fixed-with-bid-${Date.now()}`, + brand_manifest: { name: 'Fixed Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-fixed-bid', + product_id: product.product_id, + budget: 5000, + pricing_option_id: pricingOption.pricing_option_id, + bid_price: 15.00, // Unnecessary for fixed pricing + }], + }) as Promise + ); + + if (result?.success) { + step.passed = true; + step.details = 'Agent ignores bid_price for fixed pricing (permissive)'; + } else if (result && !result.success && result.error) { + step.passed = true; + step.details = 'Agent rejects bid_price for fixed pricing (strict)'; + } else { + step.passed = false; + step.error = 'Unclear response for fixed pricing with bid_price'; + } + steps.push(step); + } + + // Test 3: Budget below min_spend_per_package + if (productsWithMinSpend.length > 0) { + const { product, pricingOption, minSpend } = productsWithMinSpend[0]; + const underBudget = minSpend * 0.5; // 50% of minimum + + const { result, step } = await runStep( + 'Budget below min_spend_per_package', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `under-min-spend-${Date.now()}`, + brand_manifest: { name: 'Min Spend Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-under-min', + product_id: product.product_id, + budget: underBudget, + pricing_option_id: pricingOption.pricing_option_id, + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = `Agent rejected budget ${underBudget} below min_spend ${minSpend}`; + } else if (result?.success) { + step.passed = false; + step.error = `Agent accepted budget ${underBudget} below min_spend ${minSpend}`; + } else { + step.passed = false; + step.error = 'Unclear response for under-min-spend budget'; + } + steps.push(step); + } + + // Test 4: Bid below floor price for auction + if (auctionProducts.length > 0) { + const { product, pricingOption } = auctionProducts[0]; + const floorPrice = pricingOption.floor_price || pricingOption.price_guidance?.floor || 0; + + if (floorPrice > 0) { + const belowFloor = floorPrice * 0.5; // 50% of floor + + const { result, step } = await runStep( + 'Bid below floor price', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `below-floor-${Date.now()}`, + brand_manifest: { name: 'Floor Test', url: 'https://test.example.com' }, + start_time: new Date(Date.now() + 86400000).toISOString(), + end_time: new Date(Date.now() + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-below-floor', + product_id: product.product_id, + budget: 5000, + pricing_option_id: pricingOption.pricing_option_id, + bid_price: belowFloor, + }], + }) as Promise + ); + + if (result && !result.success && result.error) { + step.passed = true; + step.details = `Agent rejected bid ${belowFloor} below floor ${floorPrice}`; + } else if (result?.success) { + // Some agents may accept and just not win auctions + step.passed = true; + step.details = `Agent accepts bid below floor (may not win auctions)`; + } else { + step.passed = false; + step.error = 'Unclear response for below-floor bid'; + } + steps.push(step); + } + } + + return { steps, profile }; +} + +/** + * Test: Behavior Analysis + * Analyzes interesting behavioral characteristics of the agent: + * - Does it require authentication for get_products? + * - Does it require brand_manifest to return products? + * - Are responses filtered based on the brief or is everything returned? + */ +async function testBehaviorAnalysis(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + + // Create authenticated client for comparison + const authClient = createTestClient(agentUrl, 'mcp', options); + const { profile, step: profileStep } = await discoverAgentProfile(authClient); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + if (!profile.tools.includes('get_products')) { + steps.push({ + step: 'Behavior analysis', + passed: false, + duration_ms: 0, + error: 'Agent does not support get_products - cannot analyze behavior', + }); + return { steps, profile }; + } + + // Test 1: Authentication requirement - call without auth token + const noAuthOptions: TestOptions = { + ...options, + auth: undefined, // Explicitly remove auth + }; + const noAuthClient = createTestClient(agentUrl, 'mcp', noAuthOptions); + + const { result: noAuthResult, step: noAuthStep } = await runStep( + 'Get products without authentication', + 'get_products', + async () => noAuthClient.executeTask('get_products', { + brief: 'Show me all available products', + brand_manifest: { name: 'Auth Test', url: 'https://test.example.com' }, + }) as Promise + ); + + if (noAuthResult?.success && noAuthResult?.data?.products) { + const products = noAuthResult.data.products as any[]; + noAuthStep.passed = true; + noAuthStep.details = `No auth required: returned ${products.length} product(s)`; + noAuthStep.response_preview = JSON.stringify({ + auth_required: false, + products_returned: products.length, + }, null, 2); + } else if (noAuthResult && !noAuthResult.success) { + noAuthStep.passed = true; + noAuthStep.details = 'Authentication required for get_products'; + noAuthStep.response_preview = JSON.stringify({ + auth_required: true, + error: noAuthResult.error, + }, null, 2); + } else { + noAuthStep.passed = false; + noAuthStep.error = 'Unclear authentication behavior'; + } + steps.push(noAuthStep); + + // Detect if auth is required - if so, subsequent tests need auth to work + const authRequired = noAuthResult && !noAuthResult.success && + noAuthResult.error?.toLowerCase().includes('auth'); + + // Test 2: Brand manifest requirement - call without brand_manifest + const { result: noBrandResult, step: noBrandStep } = await runStep( + 'Get products without brand_manifest', + 'get_products', + async () => authClient.executeTask('get_products', { + brief: 'Show me all available products', + // Intentionally no brand_manifest + }) as Promise + ); + + if (noBrandResult?.success && noBrandResult?.data?.products) { + const products = noBrandResult.data.products as any[]; + noBrandStep.passed = true; + noBrandStep.details = `Brand manifest not required: returned ${products.length} product(s)`; + noBrandStep.response_preview = JSON.stringify({ + brand_manifest_required: false, + products_returned: products.length, + }, null, 2); + } else if (noBrandResult && !noBrandResult.success) { + // Check if this is an auth error (same as the no-auth test) or a brand_manifest error + const isAuthError = noBrandResult.error?.toLowerCase().includes('auth'); + if (isAuthError && authRequired) { + noBrandStep.passed = true; + noBrandStep.details = 'Inconclusive: auth failed (cannot test brand_manifest requirement independently)'; + noBrandStep.response_preview = JSON.stringify({ + brand_manifest_required: 'unknown', + reason: 'Auth token not accepted - cannot isolate brand_manifest requirement', + error: noBrandResult.error, + }, null, 2); + } else { + noBrandStep.passed = true; + noBrandStep.details = 'Brand manifest required for get_products'; + noBrandStep.response_preview = JSON.stringify({ + brand_manifest_required: true, + error: noBrandResult.error, + }, null, 2); + } + } else { + noBrandStep.passed = false; + noBrandStep.error = 'Unclear brand_manifest requirement'; + } + steps.push(noBrandStep); + + // Test 3: Brief relevance - compare generic vs specific briefs + // Skip these tests if auth is required but failing + let genericProductCount = 0; + let specificProductCount = 0; + let briefTestsSkipped = false; + + if (authRequired && noBrandResult && !noBrandResult.success && + noBrandResult.error?.toLowerCase().includes('auth')) { + // Auth is not working - skip brief relevance tests + briefTestsSkipped = true; + const skipStep: TestStepResult = { + step: 'Brief relevance tests', + passed: true, + duration_ms: 0, + details: 'Skipped: auth token not accepted by agent - cannot test brief filtering', + response_preview: JSON.stringify({ + skipped: true, + reason: 'Auth required but token not accepted for get_products', + }, null, 2), + }; + steps.push(skipStep); + } else { + const { result: genericResult, step: genericStep } = await runStep( + 'Get products with generic brief', + 'get_products', + async () => authClient.executeTask('get_products', { + brief: 'Show me all available advertising products', + brand_manifest: { name: 'Brief Test', url: 'https://test.example.com' }, + }) as Promise + ); + + if (genericResult?.success && genericResult?.data?.products) { + const products = genericResult.data.products as any[]; + genericProductCount = products.length; + genericStep.passed = true; + genericStep.details = `Generic brief returned ${products.length} product(s)`; + } else { + genericStep.passed = false; + genericStep.error = genericResult?.error || 'Failed to get products with generic brief'; + } + steps.push(genericStep); + + // Now try a specific brief and compare + const { result: specificResult, step: specificStep } = await runStep( + 'Get products with specific brief', + 'get_products', + async () => authClient.executeTask('get_products', { + brief: 'I need video advertising products for automotive brands targeting luxury car buyers aged 35-55', + brand_manifest: { + name: 'Luxury Auto Brand', + url: 'https://test.example.com', + industry: 'automotive', + target_audience: 'luxury car buyers aged 35-55', + }, + }) as Promise + ); + + if (specificResult?.success && specificResult?.data?.products) { + const products = specificResult.data.products as any[]; + specificProductCount = products.length; + specificStep.passed = true; + specificStep.details = `Specific brief returned ${products.length} product(s)`; + + // Analyze if results are filtered + const channels = new Set(); + for (const product of products) { + if (product.channels) { + for (const ch of product.channels) { + channels.add(ch); + } + } + } + specificStep.response_preview = JSON.stringify({ + products_returned: products.length, + channels: Array.from(channels), + }, null, 2); + } else { + specificStep.passed = false; + specificStep.error = specificResult?.error || 'Failed to get products with specific brief'; + } + steps.push(specificStep); + } + + // Test 4: Analyze filtering behavior based on comparison (skip if auth failed) + if (!briefTestsSkipped) { + const filteringAnalysisStep: TestStepResult = { + step: 'Analyze brief relevance filtering', + passed: true, + duration_ms: 0, + }; + + if (genericProductCount > 0 && specificProductCount > 0) { + if (specificProductCount < genericProductCount) { + filteringAnalysisStep.details = `Agent filters by brief: generic=${genericProductCount}, specific=${specificProductCount} (${Math.round((1 - specificProductCount / genericProductCount) * 100)}% reduction)`; + filteringAnalysisStep.response_preview = JSON.stringify({ + filtering_behavior: 'filtered', + generic_count: genericProductCount, + specific_count: specificProductCount, + reduction_percent: Math.round((1 - specificProductCount / genericProductCount) * 100), + }, null, 2); + } else if (specificProductCount === genericProductCount) { + filteringAnalysisStep.details = `Agent returns same products regardless of brief (${genericProductCount} products)`; + filteringAnalysisStep.response_preview = JSON.stringify({ + filtering_behavior: 'unfiltered', + generic_count: genericProductCount, + specific_count: specificProductCount, + note: 'Same products returned for different briefs', + }, null, 2); + } else { + filteringAnalysisStep.details = `Specific brief returned more products (${specificProductCount} > ${genericProductCount})`; + filteringAnalysisStep.response_preview = JSON.stringify({ + filtering_behavior: 'expanded', + generic_count: genericProductCount, + specific_count: specificProductCount, + note: 'More products for detailed brief - may include related products', + }, null, 2); + } + } else if (genericProductCount === 0 && specificProductCount === 0) { + filteringAnalysisStep.details = 'No products returned for either brief'; + filteringAnalysisStep.passed = false; + filteringAnalysisStep.error = 'Cannot analyze filtering - no products available'; + } else { + filteringAnalysisStep.details = `Partial results: generic=${genericProductCount}, specific=${specificProductCount}`; + } + steps.push(filteringAnalysisStep); + } + + // Test 5: Channel filtering - test if agent filters by requested channel + // Skip if auth is not working + if (!briefTestsSkipped) { + const { result: channelResult, step: channelStep } = await runStep( + 'Get products with channel-specific brief', + 'get_products', + async () => authClient.executeTask('get_products', { + brief: 'I only want display advertising products, no video or audio', + brand_manifest: { name: 'Channel Test', url: 'https://test.example.com' }, + channels: ['display'], // Explicit channel filter if supported + }) as Promise + ); + + if (channelResult?.success && channelResult?.data?.products) { + const products = channelResult.data.products as any[]; + const channels = new Set(); + for (const product of products) { + if (product.channels) { + for (const ch of product.channels) { + channels.add(ch); + } + } + } + + const hasOnlyDisplay = channels.size === 1 && channels.has('display'); + const hasDisplay = channels.has('display'); + + if (hasOnlyDisplay) { + channelStep.details = `Agent correctly filtered to display-only: ${products.length} product(s)`; + } else if (hasDisplay && channels.size > 1) { + channelStep.details = `Agent included display + other channels: ${Array.from(channels).join(', ')}`; + } else { + channelStep.details = `Agent returned channels: ${Array.from(channels).join(', ')} (${products.length} products)`; + } + channelStep.response_preview = JSON.stringify({ + products_returned: products.length, + channels_in_response: Array.from(channels), + display_only: hasOnlyDisplay, + }, null, 2); + } else if (channelResult && !channelResult.success) { + channelStep.passed = true; + channelStep.details = 'Channel filter parameter not supported'; + channelStep.response_preview = JSON.stringify({ error: channelResult.error }, null, 2); + } else { + channelStep.passed = false; + channelStep.error = 'Unclear channel filtering behavior'; + } + steps.push(channelStep); + } + + return { steps, profile }; +} + +/** + * Test: Temporal Validation + * Tests date/time ordering, format validation, and deadline logic + */ +async function testTemporalValidation(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed || !profile.tools.includes('create_media_buy')) { + steps.push({ + step: 'Temporal validation', + passed: false, + duration_ms: 0, + error: 'Agent does not support create_media_buy', + }); + return { steps, profile }; + } + + // Test 1: End time before start time (MUST fail) + const now = Date.now(); + const { result: endBeforeStartResult, step: endBeforeStartStep } = await runStep( + 'End time before start time', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `temporal-test-${Date.now()}`, + brand_manifest: { name: 'Temporal Test', url: 'https://test.example.com' }, + start_time: new Date(now + 604800000).toISOString(), // 7 days from now + end_time: new Date(now + 86400000).toISOString(), // 1 day from now (BEFORE start!) + packages: [{ + buyer_ref: 'pkg-temporal', + product_id: 'test-product', + budget: 1000, + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (endBeforeStartResult && !endBeforeStartResult.success && endBeforeStartResult.error) { + endBeforeStartStep.passed = true; + endBeforeStartStep.details = 'Agent correctly rejected end_time before start_time'; + } else if (endBeforeStartResult?.success) { + endBeforeStartStep.passed = false; + endBeforeStartStep.error = 'CRITICAL: Agent accepted end_time before start_time'; + } else { + endBeforeStartStep.passed = false; + endBeforeStartStep.error = 'Unclear response for invalid temporal ordering'; + } + steps.push(endBeforeStartStep); + + // Test 2: Start time in the past + const { result: pastStartResult, step: pastStartStep } = await runStep( + 'Start time in the past', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `past-start-${Date.now()}`, + brand_manifest: { name: 'Past Start Test', url: 'https://test.example.com' }, + start_time: new Date(now - 604800000).toISOString(), // 7 days ago (IN THE PAST!) + end_time: new Date(now + 604800000).toISOString(), // 7 days from now + packages: [{ + buyer_ref: 'pkg-past', + product_id: 'test-product', + budget: 1000, + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (pastStartResult && !pastStartResult.success && pastStartResult.error) { + pastStartStep.passed = true; + pastStartStep.details = 'Agent rejected start_time in the past'; + } else if (pastStartResult?.success) { + // Some agents may allow past start times for immediate activation + pastStartStep.passed = true; + pastStartStep.details = 'Agent accepts past start_time (immediate activation mode)'; + } else { + pastStartStep.passed = false; + pastStartStep.error = 'Unclear response for past start_time'; + } + steps.push(pastStartStep); + + // Test 3: Invalid ISO 8601 date format + const { result: invalidDateResult, step: invalidDateStep } = await runStep( + 'Invalid date format', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `invalid-date-${Date.now()}`, + brand_manifest: { name: 'Invalid Date Test', url: 'https://test.example.com' }, + start_time: '01/15/2025', // Invalid - should be ISO 8601 + end_time: new Date(now + 604800000).toISOString(), + packages: [{ + buyer_ref: 'pkg-invalid-date', + product_id: 'test-product', + budget: 1000, + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (invalidDateResult && !invalidDateResult.success && invalidDateResult.error) { + invalidDateStep.passed = true; + invalidDateStep.details = 'Agent rejected invalid date format'; + } else if (invalidDateResult?.success) { + invalidDateStep.passed = false; + invalidDateStep.error = 'Agent accepted non-ISO 8601 date format'; + } else { + invalidDateStep.passed = false; + invalidDateStep.error = 'Unclear response for invalid date format'; + } + steps.push(invalidDateStep); + + // Test 4: Very long campaign duration (edge case) + const { result: longCampaignResult, step: longCampaignStep } = await runStep( + 'Very long campaign duration (365 days)', + 'create_media_buy', + async () => client.executeTask('create_media_buy', { + buyer_ref: `long-campaign-${Date.now()}`, + brand_manifest: { name: 'Long Campaign Test', url: 'https://test.example.com' }, + start_time: new Date(now + 86400000).toISOString(), + end_time: new Date(now + 365 * 86400000).toISOString(), // 365 days! + packages: [{ + buyer_ref: 'pkg-long', + product_id: 'test-product', + budget: 100000, + pricing_option_id: 'test-pricing', + }], + }) as Promise + ); + + if (longCampaignResult?.success) { + longCampaignStep.passed = true; + longCampaignStep.details = 'Agent accepts 365-day campaign'; + } else if (longCampaignResult && !longCampaignResult.success && longCampaignResult.error) { + longCampaignStep.passed = true; + longCampaignStep.details = 'Agent has maximum campaign duration limit'; + } else { + longCampaignStep.passed = false; + longCampaignStep.error = 'Unclear response for long campaign'; + } + steps.push(longCampaignStep); + + return { steps, profile }; +} + +/** + * Test: Response Consistency + * Tests for schema errors, pagination bugs, data mismatches between fields + */ +async function testResponseConsistency(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { + const steps: TestStepResult[] = []; + const client = createTestClient(agentUrl, 'mcp', options); + + const { profile, step: profileStep } = await discoverAgentProfile(client); + steps.push(profileStep); + + if (!profileStep.passed) { + return { steps, profile }; + } + + // Test 1: list_creatives pagination consistency + if (profile.tools.includes('list_creatives')) { + const { result, step } = await runStep( + 'list_creatives pagination consistency', + 'list_creatives', + async () => client.executeTask('list_creatives', {}) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const creatives = data.creatives || []; + const querySummary = data.query_summary; + const pagination = data.pagination; + + const issues: string[] = []; + + // Check: total_matching vs returned vs creatives.length + if (querySummary) { + const totalMatching = querySummary.total_matching; + const returned = querySummary.returned; + + if (totalMatching !== undefined && returned !== undefined) { + if (totalMatching > 0 && returned === 0 && creatives.length === 0) { + issues.push(`total_matching=${totalMatching} but returned=0 and creatives array empty`); + } + if (returned !== creatives.length) { + issues.push(`returned=${returned} doesn't match creatives.length=${creatives.length}`); + } + } + } + + // Check: pagination consistency + if (pagination) { + if (pagination.has_more && creatives.length === 0) { + issues.push(`has_more=true but no creatives returned`); + } + if (pagination.total_pages !== undefined && pagination.current_page !== undefined) { + if (pagination.current_page > pagination.total_pages) { + issues.push(`current_page=${pagination.current_page} > total_pages=${pagination.total_pages}`); + } + } + } + + if (issues.length > 0) { + step.passed = false; + step.error = `Pagination inconsistencies: ${issues.join('; ')}`; + step.response_preview = JSON.stringify({ + issues, + query_summary: querySummary, + pagination, + creatives_count: creatives.length, + }, null, 2); + } else { + step.details = `Pagination consistent: ${creatives.length} creative(s)`; + step.response_preview = JSON.stringify({ + creatives_count: creatives.length, + total_matching: querySummary?.total_matching, + returned: querySummary?.returned, + }, null, 2); + } + } else if (result && !result.success) { + // Schema validation error - report it + step.passed = false; + step.error = result.error || 'list_creatives failed'; + } + steps.push(step); + } + + // Test 2: get_products response consistency + if (profile.tools.includes('get_products')) { + const { result, step } = await runStep( + 'get_products response consistency', + 'get_products', + async () => client.executeTask('get_products', { + brief: 'Show all available products', + brand_manifest: { name: 'Consistency Test', url: 'https://test.example.com' }, + }) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const products = data.products || []; + const issues: string[] = []; + + for (let i = 0; i < products.length; i++) { + const product = products[i]; + + // Check: product_id is present and non-empty + if (!product.product_id) { + issues.push(`Product[${i}] missing product_id`); + } + + // Check: pricing_options consistency + if (product.pricing_options) { + for (let j = 0; j < product.pricing_options.length; j++) { + const po = product.pricing_options[j]; + if (!po.pricing_option_id) { + issues.push(`Product[${i}].pricing_options[${j}] missing pricing_option_id`); + } + // Auction pricing should have floor_price or price_guidance + if (po.is_fixed === false && !po.floor_price && !po.price_guidance) { + issues.push(`Product[${i}].pricing_options[${j}] is auction but missing floor_price/price_guidance`); + } + } + } + + // Check: format_ids are structured objects (not plain strings) + if (product.format_ids) { + for (let j = 0; j < product.format_ids.length; j++) { + const fid = product.format_ids[j]; + if (typeof fid === 'string') { + issues.push(`Product[${i}].format_ids[${j}] is string, should be {agent_url, id} object`); + } else if (!fid.id) { + issues.push(`Product[${i}].format_ids[${j}] missing id field`); + } + } + } + } + + if (issues.length > 0) { + step.passed = false; + step.error = `Product data inconsistencies (${issues.length} issue(s))`; + step.response_preview = JSON.stringify({ + issues: issues.slice(0, 10), // Limit to first 10 + total_issues: issues.length, + products_count: products.length, + }, null, 2); + } else { + step.details = `Products consistent: ${products.length} product(s)`; + step.response_preview = JSON.stringify({ + products_count: products.length, + all_have_product_id: true, + all_pricing_options_valid: true, + }, null, 2); + } + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'get_products failed'; + } + steps.push(step); + } + + // Test 3: list_creative_formats response consistency + if (profile.tools.includes('list_creative_formats')) { + const { result, step } = await runStep( + 'list_creative_formats response consistency', + 'list_creative_formats', + async () => client.executeTask('list_creative_formats', {}) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const formatIds = data.format_ids || []; + const formats = data.formats || []; + const issues: string[] = []; + + // Check format_ids structure + for (let i = 0; i < formatIds.length; i++) { + const fid = formatIds[i]; + if (typeof fid === 'string') { + issues.push(`format_ids[${i}] is string "${fid}", should be {agent_url, id} object`); + } else if (fid && !fid.id) { + issues.push(`format_ids[${i}] missing id field`); + } + } + + // Check formats structure if present + for (let i = 0; i < formats.length; i++) { + const fmt = formats[i]; + if (!fmt.format_id?.id) { + issues.push(`formats[${i}] missing format_id.id`); + } + } + + if (issues.length > 0) { + step.passed = false; + step.error = `Format data inconsistencies (${issues.length} issue(s))`; + step.response_preview = JSON.stringify({ + issues: issues.slice(0, 10), + total_issues: issues.length, + format_ids_count: formatIds.length, + formats_count: formats.length, + }, null, 2); + } else { + step.details = `Formats consistent: ${formatIds.length} format_ids, ${formats.length} formats`; + } + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'list_creative_formats failed'; + } + steps.push(step); + } + + // Test 4: list_authorized_properties response consistency + if (profile.tools.includes('list_authorized_properties')) { + const { result, step } = await runStep( + 'list_authorized_properties response consistency', + 'list_authorized_properties', + async () => client.executeTask('list_authorized_properties', {}) as Promise + ); + + if (result?.success && result?.data) { + const data = result.data as any; + const properties = data.authorized_properties || data.properties || []; + const publisherDomains = data.publisher_domains || []; + const issues: string[] = []; + + // Check publisher_domains are strings + for (let i = 0; i < publisherDomains.length; i++) { + if (typeof publisherDomains[i] !== 'string') { + issues.push(`publisher_domains[${i}] is not a string`); + } + } + + // Check properties have required fields + for (let i = 0; i < properties.length; i++) { + const prop = properties[i]; + if (!prop.name && !prop.property_id && !prop.domain) { + issues.push(`properties[${i}] missing identifying field (name, property_id, or domain)`); + } + } + + if (issues.length > 0) { + step.passed = false; + step.error = `Property data inconsistencies (${issues.length} issue(s))`; + step.response_preview = JSON.stringify({ + issues: issues.slice(0, 10), + total_issues: issues.length, + properties_count: properties.length, + publisher_domains_count: publisherDomains.length, + }, null, 2); + } else { + step.details = `Properties consistent: ${properties.length} properties, ${publisherDomains.length} domains`; + } + } else if (result && !result.success) { + step.passed = false; + step.error = result.error || 'list_authorized_properties failed'; + } + steps.push(step); + } + + return { steps, profile }; +} + +/** + * Main entry point: Run a test scenario against an agent + */ +export async function testAgent( + agentUrl: string, + scenario: TestScenario, + options: TestOptions = {} +): Promise { + const startTime = Date.now(); + let steps: TestStepResult[] = []; + let profile: AgentProfile | undefined; + + // Default dry_run to true for safety + const effectiveOptions = { + ...options, + dry_run: options.dry_run !== false, + test_session_id: options.test_session_id || `addie-test-${Date.now()}`, + }; + + logger.info({ agentUrl, scenario, options: effectiveOptions }, 'Starting agent test'); + + try { + let result: { steps: TestStepResult[]; profile?: AgentProfile }; + + switch (scenario) { + case 'health_check': + steps = await testHealthCheck(agentUrl, effectiveOptions); + break; + case 'discovery': + result = await testDiscovery(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'create_media_buy': + result = await testCreateMediaBuy(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'full_sales_flow': + result = await testFullSalesFlow(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'creative_sync': + result = await testCreativeSync(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'creative_inline': + result = await testCreativeInline(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'creative_reference': + // TODO: Implement reference creative testing + steps = [{ + step: 'Test reference creatives', + passed: false, + duration_ms: 0, + error: 'creative_reference scenario not yet implemented', + }]; + break; + case 'pricing_models': + result = await testPricingModels(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'creative_flow': + result = await testCreativeFlow(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'signals_flow': + result = await testSignalsFlow(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'error_handling': + result = await testErrorHandling(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'validation': + result = await testValidation(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'pricing_edge_cases': + result = await testPricingEdgeCases(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'temporal_validation': + result = await testTemporalValidation(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'behavior_analysis': + result = await testBehaviorAnalysis(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + case 'response_consistency': + result = await testResponseConsistency(agentUrl, effectiveOptions); + steps = result.steps; + profile = result.profile; + break; + default: + steps = [{ + step: 'Unknown scenario', + passed: false, + duration_ms: 0, + error: `Unknown test scenario: ${scenario}`, + }]; + } + } catch (error) { + logger.error({ error, agentUrl, scenario }, 'Agent test failed with exception'); + steps.push({ + step: 'Test execution', + passed: false, + duration_ms: Date.now() - startTime, + error: error instanceof Error ? error.message : String(error), + }); + } + + const totalDuration = Date.now() - startTime; + const passedCount = steps.filter(s => s.passed).length; + const failedCount = steps.filter(s => !s.passed).length; + const overallPassed = failedCount === 0 && passedCount > 0; + + // Generate summary + let summary: string; + if (overallPassed) { + summary = `All ${passedCount} test step(s) passed in ${totalDuration}ms`; + } else if (passedCount === 0) { + summary = `All ${failedCount} test step(s) failed`; + } else { + summary = `${passedCount} passed, ${failedCount} failed out of ${steps.length} step(s)`; + } + + const testResult: TestResult = { + agent_url: agentUrl, + scenario, + overall_passed: overallPassed, + steps, + summary, + total_duration_ms: totalDuration, + tested_at: new Date().toISOString(), + agent_profile: profile, + dry_run: effectiveOptions.dry_run, + }; + + logger.info({ agentUrl, scenario, overallPassed, passedCount, failedCount, totalDuration }, 'Agent test completed'); + + return testResult; +} + +/** + * Format test results for display in Slack/chat + */ +export function formatTestResults(result: TestResult): string { + const statusEmoji = result.overall_passed ? '✅' : '❌'; + let output = `## ${statusEmoji} Agent Test Results\n\n`; + output += `**Agent:** ${result.agent_url}\n`; + output += `**Scenario:** ${result.scenario}\n`; + output += `**Duration:** ${result.total_duration_ms}ms\n`; + output += `**Mode:** ${result.dry_run ? '🧪 Dry Run' : '🔴 Live'}\n`; + output += `**Result:** ${result.summary}\n\n`; + + // Show agent profile if discovered + if (result.agent_profile) { + output += `### Agent Capabilities\n`; + output += `- **Name:** ${result.agent_profile.name}\n`; + output += `- **Tools:** ${result.agent_profile.tools.length}\n`; + if (result.agent_profile.channels?.length) { + output += `- **Channels:** ${result.agent_profile.channels.join(', ')}\n`; + } + if (result.agent_profile.pricing_models?.length) { + output += `- **Pricing Models:** ${result.agent_profile.pricing_models.join(', ')}\n`; + } + output += '\n'; + } + + output += `### Test Steps\n\n`; + + for (const step of result.steps) { + const stepEmoji = step.passed ? '✅' : '❌'; + output += `${stepEmoji} **${step.step}**`; + if (step.task) { + output += ` (\`${step.task}\`)`; + } + output += ` - ${step.duration_ms}ms\n`; + + if (step.details) { + output += ` ${step.details}\n`; + } + + if (step.error) { + output += ` ⚠️ Error: ${step.error}\n`; + } + + if (step.response_preview && !step.error) { + output += ` \`\`\`json\n ${step.response_preview.split('\n').join('\n ')}\n \`\`\`\n`; + } + + output += '\n'; + } + + if (!result.overall_passed) { + output += `---\n\n`; + output += `💡 **Need help?** Ask me about specific errors or check the [AdCP documentation](https://adcontextprotocol.org/docs).\n`; + } + + return output; +} diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index fdfe1ef437..a8589c1ea3 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -15,8 +15,11 @@ import { logger } from '../../logger.js'; import type { AddieTool } from '../types.js'; import { AdAgentsManager } from '../../adagents-manager.js'; import type { MemberContext } from '../member-context.js'; +import { testAgent, formatTestResults, type TestScenario, type TestOptions } from './agent-tester.js'; +import { AgentContextDatabase } from '../../db/agent-context-db.js'; const adagentsManager = new AdAgentsManager(); +const agentContextDb = new AgentContextDatabase(); /** * Tool definitions for member-related operations @@ -287,6 +290,130 @@ export const MEMBER_TOOLS: AddieTool[] = [ required: ['agent_url'], }, }, + { + name: 'test_adcp_agent', + description: + 'Run end-to-end tests against an AdCP agent to verify it works correctly. Tests the full workflow: discover products, create media buys, sync creatives, etc. By default runs in dry-run mode - set dry_run=false for real testing. Use this when users want to test their agent implementation, verify compliance, or debug issues. This replaces the testing.adcontextprotocol.org harness.', + usage_hints: 'use for "test my agent", "run the full flow", "verify my sales agent works", "test against test-agent", "test creative sync", "test pricing models"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to test (e.g., "https://sales.example.com" or "https://test-agent.adcontextprotocol.org")', + }, + scenario: { + type: 'string', + enum: [ + 'health_check', + 'discovery', + 'create_media_buy', + 'full_sales_flow', + 'creative_sync', + 'creative_inline', + 'creative_reference', + 'pricing_models', + 'creative_flow', + 'signals_flow', + 'error_handling', + 'validation', + 'pricing_edge_cases', + 'temporal_validation', + 'behavior_analysis', + 'response_consistency', + ], + description: 'Test scenario: health_check (agent responds), discovery (products/formats/properties), create_media_buy (discovery + create), full_sales_flow (create + update + delivery), creative_sync (sync_creatives flow), creative_inline (inline creatives in create_media_buy), creative_reference (reference existing creatives), pricing_models (analyze pricing options), creative_flow (creative agents), signals_flow (signals agents), error_handling (proper error responses), validation (invalid input rejection), pricing_edge_cases (auction vs fixed, min spend), temporal_validation (date ordering, format), behavior_analysis (auth requirements, brief relevance, filtering behavior), response_consistency (schema errors, pagination bugs, data mismatches)', + }, + brief: { + type: 'string', + description: 'Optional custom brief for product discovery (default: generic tech brand brief)', + }, + budget: { + type: 'number', + description: 'Budget for test media buy in dollars (default: 1000)', + }, + dry_run: { + type: 'boolean', + description: 'Whether to run in dry-run mode (default: true). Set to false for real testing that creates actual media buys.', + }, + channels: { + type: 'array', + items: { type: 'string' }, + description: 'Specific channels to test (e.g., ["display", "video", "ctv"]). If not specified, tests all channels the agent supports.', + }, + pricing_models: { + type: 'array', + items: { type: 'string' }, + description: 'Specific pricing models to test (e.g., ["cpm", "cpcv"]). If not specified, uses first available.', + }, + auth_token: { + type: 'string', + description: 'Bearer token for agents that require authentication. For test-agent.adcontextprotocol.org, use the published test credentials.', + }, + }, + required: ['agent_url'], + }, + }, + + // ============================================ + // AGENT CONTEXT MANAGEMENT + // ============================================ + { + name: 'save_agent', + description: + 'Save an agent URL to the organization\'s context. Optionally store an auth token securely (encrypted, never shown in conversations). Use this when users want to save their agent for easy testing later, or when they provide an auth token.', + usage_hints: 'use for "save my agent", "remember this agent URL", "store my auth token"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to save (e.g., "https://sales.example.com/mcp")', + }, + agent_name: { + type: 'string', + description: 'Friendly name for the agent (e.g., "Production Sales Agent")', + }, + auth_token: { + type: 'string', + description: 'Optional auth token to store securely. Will be encrypted and never shown again.', + }, + protocol: { + type: 'string', + enum: ['mcp', 'a2a'], + description: 'Protocol type (default: mcp)', + }, + }, + required: ['agent_url'], + }, + }, + { + name: 'list_saved_agents', + description: + 'List all agents saved for this organization. Shows agent URLs, names, types, and whether they have auth tokens stored (but never shows the actual tokens). Use this when users ask "what agents do I have saved?" or want to see their configured agents.', + usage_hints: 'use for "show my agents", "what agents are saved?", "list our agents"', + input_schema: { + type: 'object', + properties: {}, + required: [], + }, + }, + { + name: 'remove_saved_agent', + description: + 'Remove a saved agent and its stored auth token. Use this when users want to delete or forget an agent configuration.', + usage_hints: 'use for "remove my agent", "delete the agent", "forget this agent"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to remove', + }, + }, + required: ['agent_url'], + }, + }, // ============================================ // GITHUB ISSUE DRAFTING @@ -994,6 +1121,104 @@ export function createMemberToolHandlers( return response; }); + // ============================================ + // E2E AGENT TESTING + // ============================================ + handlers.set('test_adcp_agent', async (input) => { + const agentUrl = input.agent_url as string; + const scenario = (input.scenario as TestScenario) || 'discovery'; + const brief = input.brief as string | undefined; + const budget = input.budget as number | undefined; + const dryRun = input.dry_run as boolean | undefined; + const channels = input.channels as string[] | undefined; + const pricingModels = input.pricing_models as string[] | undefined; + let authToken = input.auth_token as string | undefined; + + // Auto-lookup saved token if user didn't provide one and has org context + let usingSavedToken = false; + const organizationId = memberContext?.organization?.workos_organization_id; + if (!authToken && organizationId) { + try { + const savedToken = await agentContextDb.getAuthTokenByOrgAndUrl( + organizationId, + agentUrl + ); + if (savedToken) { + authToken = savedToken; + usingSavedToken = true; + logger.info({ agentUrl }, 'Using saved auth token for agent test'); + } + } catch (error) { + // Non-fatal - continue without saved token + logger.debug({ error, agentUrl }, 'Could not lookup saved token'); + } + } + + const options: TestOptions = { + test_session_id: `addie-test-${Date.now()}`, + dry_run: dryRun, // undefined means default to true + }; + if (brief) options.brief = brief; + if (budget) options.budget = budget; + if (channels) options.channels = channels; + if (pricingModels) options.pricing_models = pricingModels; + if (authToken) options.auth = { type: 'bearer', token: authToken }; + + try { + const result = await testAgent(agentUrl, scenario, options); + + // If user is authenticated and agent test succeeded, update the saved context + if (organizationId) { + try { + const context = await agentContextDb.getByOrgAndUrl( + organizationId, + agentUrl + ); + if (context && result.agent_profile) { + // Update with discovered tools and test results + const tools = result.agent_profile.tools || []; + await agentContextDb.update(context.id, { + tools_discovered: tools, + agent_type: agentContextDb.inferAgentType(tools), + last_test_scenario: scenario, + last_test_passed: result.overall_passed, + last_test_summary: result.summary, + }); + + // Record test history + await agentContextDb.recordTest({ + agent_context_id: context.id, + scenario, + overall_passed: result.overall_passed, + steps_passed: result.steps.filter((s) => s.passed).length, + steps_failed: result.steps.filter((s) => !s.passed).length, + total_duration_ms: result.total_duration_ms, + summary: result.summary, + dry_run: options.dry_run !== false, + brief: options.brief, + triggered_by: 'user', + user_id: memberContext?.workos_user?.workos_user_id, + steps_json: result.steps, + agent_profile_json: result.agent_profile, + }); + } + } catch (error) { + // Non-fatal - test still ran + logger.debug({ error }, 'Could not update agent context after test'); + } + } + + let output = formatTestResults(result); + if (usingSavedToken) { + output = `_Using saved credentials for this agent._\n\n` + output; + } + return output; + } catch (error) { + logger.error({ error, agentUrl, scenario }, 'Addie: test_adcp_agent failed'); + return `Failed to test agent ${agentUrl}: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + // ============================================ // GITHUB ISSUE DRAFTING // ============================================ @@ -1054,5 +1279,172 @@ export function createMemberToolHandlers( return response; }); + // ============================================ + // AGENT CONTEXT MANAGEMENT + // ============================================ + handlers.set('save_agent', async (input) => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to save agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const saveOrgId = memberContext.organization?.workos_organization_id; + if (!saveOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + const agentUrl = input.agent_url as string; + const agentName = input.agent_name as string | undefined; + const authToken = input.auth_token as string | undefined; + const protocol = (input.protocol as 'mcp' | 'a2a') || 'mcp'; + + try { + // Check if agent already exists for this org + let context = await agentContextDb.getByOrgAndUrl(saveOrgId, agentUrl); + + if (context) { + // Update existing context + if (agentName) { + await agentContextDb.update(context.id, { agent_name: agentName, protocol }); + } + if (authToken) { + await agentContextDb.saveAuthToken(context.id, authToken); + } + // Refresh context + context = await agentContextDb.getById(context.id); + + let response = `✅ Updated saved agent: **${context?.agent_name || agentUrl}**\n\n`; + if (authToken) { + response += `🔐 Auth token saved securely (hint: ${context?.auth_token_hint})\n`; + response += `_The token is encrypted and will never be shown again._\n`; + } + return response; + } + + // Create new context + context = await agentContextDb.create({ + organization_id: saveOrgId, + agent_url: agentUrl, + agent_name: agentName, + protocol, + created_by: memberContext.workos_user.workos_user_id, + }); + + // Save auth token if provided + if (authToken) { + await agentContextDb.saveAuthToken(context.id, authToken); + context = await agentContextDb.getById(context.id); + } + + let response = `✅ Saved agent: **${context?.agent_name || agentUrl}**\n\n`; + response += `**URL:** ${agentUrl}\n`; + response += `**Protocol:** ${protocol.toUpperCase()}\n`; + if (authToken) { + response += `\n🔐 Auth token saved securely (hint: ${context?.auth_token_hint})\n`; + response += `_The token is encrypted and will never be shown again._\n`; + } + response += `\nWhen you test this agent, I'll automatically use the saved credentials.`; + + return response; + } catch (error) { + logger.error({ error, agentUrl }, 'Addie: save_agent failed'); + return `Failed to save agent: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + + handlers.set('list_saved_agents', async () => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to list saved agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const listOrgId = memberContext.organization?.workos_organization_id; + if (!listOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + try { + const agents = await agentContextDb.getByOrganization(listOrgId); + + if (agents.length === 0) { + return 'No agents saved yet. Use `save_agent` to save an agent URL for easy testing.'; + } + + let response = `## Your Saved Agents\n\n`; + + for (const agent of agents) { + const name = agent.agent_name || 'Unnamed Agent'; + const type = agent.agent_type !== 'unknown' ? ` (${agent.agent_type})` : ''; + const hasToken = agent.has_auth_token ? `🔐 ${agent.auth_token_hint}` : '🔓 No token'; + + response += `### ${name}${type}\n`; + response += `**URL:** ${agent.agent_url}\n`; + response += `**Protocol:** ${agent.protocol.toUpperCase()}\n`; + response += `**Auth:** ${hasToken}\n`; + + if (agent.tools_discovered && agent.tools_discovered.length > 0) { + response += `**Tools:** ${agent.tools_discovered.slice(0, 5).join(', ')}`; + if (agent.tools_discovered.length > 5) { + response += ` (+${agent.tools_discovered.length - 5} more)`; + } + response += `\n`; + } + + if (agent.last_tested_at) { + const lastTest = new Date(agent.last_tested_at).toLocaleDateString(); + const status = agent.last_test_passed ? '✅' : '❌'; + response += `**Last Test:** ${status} ${agent.last_test_scenario} (${lastTest})\n`; + response += `**Total Tests:** ${agent.total_tests_run}\n`; + } + + response += `\n`; + } + + return response; + } catch (error) { + logger.error({ error }, 'Addie: list_saved_agents failed'); + return `Failed to list agents: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + + handlers.set('remove_saved_agent', async (input) => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to remove saved agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const removeOrgId = memberContext.organization?.workos_organization_id; + if (!removeOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + const agentUrl = input.agent_url as string; + + try { + // Find the agent + const context = await agentContextDb.getByOrgAndUrl(removeOrgId, agentUrl); + + if (!context) { + return `No saved agent found with URL: ${agentUrl}\n\nUse \`list_saved_agents\` to see your saved agents.`; + } + + const agentName = context.agent_name || agentUrl; + + // Delete it + await agentContextDb.delete(context.id); + + let response = `✅ Removed saved agent: **${agentName}**\n\n`; + if (context.has_auth_token) { + response += `🔐 The stored auth token has been permanently deleted.\n`; + } + response += `All test history for this agent has also been removed.`; + + return response; + } catch (error) { + logger.error({ error, agentUrl }, 'Addie: remove_saved_agent failed'); + return `Failed to remove agent: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + return handlers; } diff --git a/server/src/db/agent-context-db.ts b/server/src/db/agent-context-db.ts new file mode 100644 index 0000000000..0f7c841347 --- /dev/null +++ b/server/src/db/agent-context-db.ts @@ -0,0 +1,534 @@ +import { query } from './client.js'; +import crypto from 'crypto'; + +// ===================================================== +// TYPES +// ===================================================== + +export type AgentType = 'sales' | 'creative' | 'signals' | 'unknown'; +export type Protocol = 'mcp' | 'a2a'; + +export interface AgentContext { + id: string; + organization_id: string; + agent_url: string; + agent_name: string | null; + agent_type: AgentType; + protocol: Protocol; + // Token info (never expose actual token!) + has_auth_token: boolean; + auth_token_hint: string | null; + // Discovery cache + tools_discovered: string[] | null; + last_discovered_at: Date | null; + // Test history + last_test_scenario: string | null; + last_test_passed: boolean | null; + last_test_summary: string | null; + last_tested_at: Date | null; + total_tests_run: number; + // Metadata + created_at: Date; + updated_at: Date; + created_by: string | null; +} + +export interface AgentTestHistory { + id: string; + agent_context_id: string; + scenario: string; + overall_passed: boolean; + steps_passed: number; + steps_failed: number; + total_duration_ms: number | null; + summary: string | null; + dry_run: boolean; + brief: string | null; + triggered_by: string | null; + user_id: string | null; + steps_json: any; + agent_profile_json: any; + started_at: Date; + completed_at: Date | null; +} + +export interface CreateAgentContextInput { + organization_id: string; + agent_url: string; + agent_name?: string; + agent_type?: AgentType; + protocol?: Protocol; + created_by?: string; +} + +export interface UpdateAgentContextInput { + agent_name?: string; + agent_type?: AgentType; + protocol?: Protocol; + tools_discovered?: string[]; + last_test_scenario?: string; + last_test_passed?: boolean; + last_test_summary?: string; +} + +export interface RecordTestInput { + agent_context_id: string; + scenario: string; + overall_passed: boolean; + steps_passed: number; + steps_failed: number; + total_duration_ms?: number; + summary?: string; + dry_run?: boolean; + brief?: string; + triggered_by?: string; + user_id?: string; + steps_json?: any; + agent_profile_json?: any; +} + +// ===================================================== +// ENCRYPTION HELPERS +// ===================================================== + +// Encryption key derivation (in production, use a proper KMS) +// For now, derive key from a secret + org ID +const ENCRYPTION_SECRET = process.env.AGENT_TOKEN_ENCRYPTION_SECRET || 'dev-secret-change-in-production'; + +function deriveKey(organizationId: string): Buffer { + return crypto.pbkdf2Sync(ENCRYPTION_SECRET, organizationId, 100000, 32, 'sha256'); +} + +function encryptToken(token: string, organizationId: string): { encrypted: string; iv: string } { + const key = deriveKey(organizationId); + const iv = crypto.randomBytes(16); + const cipher = crypto.createCipheriv('aes-256-gcm', key, iv); + + let encrypted = cipher.update(token, 'utf8', 'base64'); + encrypted += cipher.final('base64'); + + // Append auth tag + const authTag = cipher.getAuthTag(); + encrypted += ':' + authTag.toString('base64'); + + return { + encrypted, + iv: iv.toString('base64'), + }; +} + +function decryptToken(encrypted: string, iv: string, organizationId: string): string { + const key = deriveKey(organizationId); + const ivBuffer = Buffer.from(iv, 'base64'); + + // Split encrypted data and auth tag + const [encryptedData, authTagBase64] = encrypted.split(':'); + const authTag = Buffer.from(authTagBase64, 'base64'); + + const decipher = crypto.createDecipheriv('aes-256-gcm', key, ivBuffer); + decipher.setAuthTag(authTag); + + let decrypted = decipher.update(encryptedData, 'base64', 'utf8'); + decrypted += decipher.final('utf8'); + + return decrypted; +} + +function getTokenHint(token: string): string { + if (token.length <= 4) return '****'; + return '****' + token.slice(-4); +} + +// ===================================================== +// AGENT CONTEXT DATABASE +// ===================================================== + +export class AgentContextDatabase { + /** + * Get all agent contexts for an organization + */ + async getByOrganization(organizationId: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE organization_id = $1 + ORDER BY updated_at DESC`, + [organizationId] + ); + return result.rows; + } + + /** + * Get a specific agent context by ID + */ + async getById(id: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE id = $1`, + [id] + ); + return result.rows[0] || null; + } + + /** + * Get agent context by organization and URL + */ + async getByOrgAndUrl(organizationId: string, agentUrl: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE organization_id = $1 AND agent_url = $2`, + [organizationId, agentUrl] + ); + return result.rows[0] || null; + } + + /** + * Create a new agent context + */ + async create(input: CreateAgentContextInput): Promise { + const result = await query( + `INSERT INTO agent_contexts ( + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + created_by + ) VALUES ($1, $2, $3, $4, $5, $6) + RETURNING + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + FALSE as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by`, + [ + input.organization_id, + input.agent_url, + input.agent_name || null, + input.agent_type || 'unknown', + input.protocol || 'mcp', + input.created_by || null, + ] + ); + return result.rows[0]; + } + + /** + * Update an agent context + */ + async update(id: string, input: UpdateAgentContextInput): Promise { + const updates: string[] = []; + const values: any[] = []; + let paramIndex = 1; + + if (input.agent_name !== undefined) { + updates.push(`agent_name = $${paramIndex++}`); + values.push(input.agent_name); + } + if (input.agent_type !== undefined) { + updates.push(`agent_type = $${paramIndex++}`); + values.push(input.agent_type); + } + if (input.protocol !== undefined) { + updates.push(`protocol = $${paramIndex++}`); + values.push(input.protocol); + } + if (input.tools_discovered !== undefined) { + updates.push(`tools_discovered = $${paramIndex++}`); + updates.push(`last_discovered_at = NOW()`); + values.push(input.tools_discovered); + } + if (input.last_test_scenario !== undefined) { + updates.push(`last_test_scenario = $${paramIndex++}`); + values.push(input.last_test_scenario); + } + if (input.last_test_passed !== undefined) { + updates.push(`last_test_passed = $${paramIndex++}`); + values.push(input.last_test_passed); + } + if (input.last_test_summary !== undefined) { + updates.push(`last_test_summary = $${paramIndex++}`); + values.push(input.last_test_summary); + } + + if (updates.length === 0) { + return this.getById(id); + } + + updates.push('updated_at = NOW()'); + values.push(id); + + const result = await query( + `UPDATE agent_contexts + SET ${updates.join(', ')} + WHERE id = $${paramIndex} + RETURNING + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by`, + values + ); + return result.rows[0] || null; + } + + /** + * Save an auth token (encrypted) + * IMPORTANT: Token is encrypted and never returned in queries + */ + async saveAuthToken(id: string, token: string): Promise { + // Get the org ID for key derivation + const context = await this.getById(id); + if (!context) { + throw new Error(`Agent context ${id} not found`); + } + + const { encrypted, iv } = encryptToken(token, context.organization_id); + const hint = getTokenHint(token); + + await query( + `UPDATE agent_contexts + SET + auth_token_encrypted = $1, + auth_token_iv = $2, + auth_token_hint = $3, + updated_at = NOW() + WHERE id = $4`, + [encrypted, iv, hint, id] + ); + } + + /** + * Get decrypted auth token (for internal use only - NEVER expose to users) + * Returns null if no token stored + */ + async getAuthToken(id: string): Promise { + const result = await query( + `SELECT organization_id, auth_token_encrypted, auth_token_iv + FROM agent_contexts + WHERE id = $1`, + [id] + ); + + const row = result.rows[0]; + if (!row || !row.auth_token_encrypted || !row.auth_token_iv) { + return null; + } + + return decryptToken(row.auth_token_encrypted, row.auth_token_iv, row.organization_id); + } + + /** + * Get auth token by org and URL (for test_adcp_agent tool) + */ + async getAuthTokenByOrgAndUrl(organizationId: string, agentUrl: string): Promise { + const result = await query( + `SELECT id, auth_token_encrypted, auth_token_iv + FROM agent_contexts + WHERE organization_id = $1 AND agent_url = $2`, + [organizationId, agentUrl] + ); + + const row = result.rows[0]; + if (!row || !row.auth_token_encrypted || !row.auth_token_iv) { + return null; + } + + return decryptToken(row.auth_token_encrypted, row.auth_token_iv, organizationId); + } + + /** + * Remove auth token + */ + async removeAuthToken(id: string): Promise { + await query( + `UPDATE agent_contexts + SET + auth_token_encrypted = NULL, + auth_token_iv = NULL, + auth_token_hint = NULL, + updated_at = NOW() + WHERE id = $1`, + [id] + ); + } + + /** + * Delete an agent context + */ + async delete(id: string): Promise { + const result = await query('DELETE FROM agent_contexts WHERE id = $1', [id]); + return (result.rowCount ?? 0) > 0; + } + + /** + * Record a test run + */ + async recordTest(input: RecordTestInput): Promise { + // Update the agent context + await query( + `UPDATE agent_contexts + SET + last_test_scenario = $1, + last_test_passed = $2, + last_test_summary = $3, + last_tested_at = NOW(), + total_tests_run = total_tests_run + 1, + updated_at = NOW() + WHERE id = $4`, + [input.scenario, input.overall_passed, input.summary || null, input.agent_context_id] + ); + + // Insert history record + const result = await query( + `INSERT INTO agent_test_history ( + agent_context_id, + scenario, + overall_passed, + steps_passed, + steps_failed, + total_duration_ms, + summary, + dry_run, + brief, + triggered_by, + user_id, + steps_json, + agent_profile_json, + completed_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW()) + RETURNING *`, + [ + input.agent_context_id, + input.scenario, + input.overall_passed, + input.steps_passed, + input.steps_failed, + input.total_duration_ms || null, + input.summary || null, + input.dry_run ?? true, + input.brief || null, + input.triggered_by || null, + input.user_id || null, + input.steps_json ? JSON.stringify(input.steps_json) : null, + input.agent_profile_json ? JSON.stringify(input.agent_profile_json) : null, + ] + ); + + return result.rows[0]; + } + + /** + * Get test history for an agent + */ + async getTestHistory(agentContextId: string, limit: number = 20): Promise { + const result = await query( + `SELECT * + FROM agent_test_history + WHERE agent_context_id = $1 + ORDER BY started_at DESC + LIMIT $2`, + [agentContextId, limit] + ); + return result.rows; + } + + /** + * Infer agent type from discovered tools + */ + inferAgentType(tools: string[]): AgentType { + if (tools.includes('get_products') || tools.includes('create_media_buy')) { + return 'sales'; + } + if (tools.includes('list_creative_formats') && !tools.includes('get_products')) { + return 'creative'; + } + if (tools.includes('get_signals') || tools.includes('activate_signal')) { + return 'signals'; + } + return 'unknown'; + } +} diff --git a/server/src/db/migrations/079_agent_contexts.sql b/server/src/db/migrations/079_agent_contexts.sql new file mode 100644 index 0000000000..e80b6af7f3 --- /dev/null +++ b/server/src/db/migrations/079_agent_contexts.sql @@ -0,0 +1,136 @@ +-- Migration: 079_agent_contexts.sql +-- Agent Testing Context System +-- +-- Stores agent URLs that users are working on, their test history, +-- and securely stored auth tokens (encrypted). + +-- ===================================================== +-- AGENT CONTEXTS TABLE +-- ===================================================== +-- Tracks agents that organizations are developing/testing + +CREATE TABLE IF NOT EXISTS agent_contexts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Organization scope + organization_id TEXT NOT NULL REFERENCES organizations(workos_organization_id), + + -- Agent identification + agent_url TEXT NOT NULL, -- The agent's MCP/A2A endpoint + agent_name TEXT, -- Friendly name like "Our Production Sales Agent" + agent_type TEXT DEFAULT 'unknown' -- 'sales' | 'creative' | 'signals' | 'unknown' + CHECK (agent_type IN ('sales', 'creative', 'signals', 'unknown')), + protocol TEXT DEFAULT 'mcp' -- 'mcp' | 'a2a' + CHECK (protocol IN ('mcp', 'a2a')), + + -- Secure token storage + -- Token is encrypted with AES-256-GCM using org-specific derived key + -- NEVER expose the actual token in responses or logs + auth_token_encrypted TEXT, -- Encrypted token (base64) + auth_token_iv TEXT, -- Initialization vector (base64) + auth_token_hint TEXT, -- Last 4 chars for display: "****ABCD" + + -- Discovery cache (updated after each test) + tools_discovered TEXT[], -- ['get_products', 'create_media_buy', ...] + last_discovered_at TIMESTAMPTZ, + + -- Test history + last_test_scenario TEXT, -- Most recent scenario run + last_test_passed BOOLEAN, -- Did it pass? + last_test_summary TEXT, -- Brief summary + last_tested_at TIMESTAMPTZ, + total_tests_run INTEGER DEFAULT 0, + + -- Metadata + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + created_by TEXT, -- WorkOS user ID who added it + + -- One agent URL per organization + UNIQUE(organization_id, agent_url) +); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_agent_contexts_org ON agent_contexts(organization_id); +CREATE INDEX IF NOT EXISTS idx_agent_contexts_type ON agent_contexts(agent_type); +CREATE INDEX IF NOT EXISTS idx_agent_contexts_updated ON agent_contexts(updated_at DESC); + +COMMENT ON TABLE agent_contexts IS 'Agent URLs and test history for each organization'; +COMMENT ON COLUMN agent_contexts.auth_token_encrypted IS 'AES-256-GCM encrypted auth token - NEVER expose'; +COMMENT ON COLUMN agent_contexts.auth_token_hint IS 'Last 4 chars of token for display (e.g., ****ABCD)'; +COMMENT ON COLUMN agent_contexts.tools_discovered IS 'Cached list of tools from last discovery'; + +-- ===================================================== +-- AGENT TEST HISTORY TABLE +-- ===================================================== +-- Detailed history of test runs (for debugging and analysis) + +CREATE TABLE IF NOT EXISTS agent_test_history ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Link to agent context + agent_context_id UUID NOT NULL REFERENCES agent_contexts(id) ON DELETE CASCADE, + + -- Test details + scenario TEXT NOT NULL, + overall_passed BOOLEAN NOT NULL, + steps_passed INTEGER NOT NULL DEFAULT 0, + steps_failed INTEGER NOT NULL DEFAULT 0, + total_duration_ms INTEGER, + summary TEXT, + + -- Options used + dry_run BOOLEAN DEFAULT TRUE, + brief TEXT, -- Custom brief if provided + + -- Who ran it + triggered_by TEXT, -- 'user' | 'scheduled' | 'api' + user_id TEXT, -- WorkOS user ID if user-triggered + + -- Results (stored as JSON for flexibility) + steps_json JSONB, -- Full step results + agent_profile_json JSONB, -- Discovered agent profile + + -- Timestamps + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_test_history_agent ON agent_test_history(agent_context_id); +CREATE INDEX IF NOT EXISTS idx_test_history_started ON agent_test_history(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_test_history_scenario ON agent_test_history(scenario); +CREATE INDEX IF NOT EXISTS idx_test_history_passed ON agent_test_history(overall_passed); + +COMMENT ON TABLE agent_test_history IS 'Detailed test run history for debugging and analysis'; +COMMENT ON COLUMN agent_test_history.steps_json IS 'Full TestStepResult[] as JSON'; + +-- ===================================================== +-- VIEW: AGENT CONTEXT SUMMARY +-- ===================================================== +-- Summary view for displaying agent contexts to users + +CREATE OR REPLACE VIEW agent_context_summary AS +SELECT + ac.id, + ac.organization_id, + ac.agent_url, + ac.agent_name, + ac.agent_type, + ac.protocol, + ac.auth_token_hint, + ac.auth_token_encrypted IS NOT NULL as has_auth_token, + ac.tools_discovered, + ac.last_test_scenario, + ac.last_test_passed, + ac.last_test_summary, + ac.last_tested_at, + ac.total_tests_run, + ac.created_at, + ac.updated_at, + -- Aggregated stats from history + (SELECT COUNT(*) FROM agent_test_history h WHERE h.agent_context_id = ac.id) as history_count, + (SELECT COUNT(*) FROM agent_test_history h WHERE h.agent_context_id = ac.id AND h.overall_passed) as history_passed_count +FROM agent_contexts ac; + +COMMENT ON VIEW agent_context_summary IS 'Agent contexts with token visibility hidden and stats aggregated'; From 3958c5636ad2879717db6c30c8c71d96246443de Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 2 Jan 2026 16:12:53 -0500 Subject: [PATCH 2/8] Fix BASE_URL port priority for Docker internal calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PORT (internal server port, 8080 in Docker) should take precedence over CONDUCTOR_PORT (external mapping) when making internal API calls. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- server/src/addie/mcp/member-tools.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index a8589c1ea3..f69190f52c 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -455,12 +455,14 @@ export const MEMBER_TOOLS: AddieTool[] = [ /** * Base URL for internal API calls * Uses BASE_URL env var in production, falls back to localhost for development + * Note: PORT takes precedence over CONDUCTOR_PORT for internal calls (inside Docker, PORT=8080) */ function getBaseUrl(): string { if (process.env.BASE_URL) { return process.env.BASE_URL; } - const port = process.env.CONDUCTOR_PORT || process.env.PORT || '3000'; + // PORT is the internal server port (8080 in Docker), CONDUCTOR_PORT is external mapping + const port = process.env.PORT || process.env.CONDUCTOR_PORT || '3000'; return `http://localhost:${port}`; } From 422fb00eb09da5224dae527ed056753cd85cf0c7 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 2 Jan 2026 16:13:23 -0500 Subject: [PATCH 3/8] Add Addie testing documentation to testing.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the new E2E agent testing capabilities: - Testing scenarios (discovery, full_sales_flow, etc.) - Edge case scenarios (error_handling, validation) - Behavioral analysis features - Sales agent compliance checklist 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- docs/media-buy/advanced-topics/testing.mdx | 85 +++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/docs/media-buy/advanced-topics/testing.mdx b/docs/media-buy/advanced-topics/testing.mdx index 03a196df95..3eb46502cf 100644 --- a/docs/media-buy/advanced-topics/testing.mdx +++ b/docs/media-buy/advanced-topics/testing.mdx @@ -50,7 +50,90 @@ AdCP provides a **public test agent** with free credentials for development and ## Protocol Compliance Testing -Use the [AdCP Protocol Test Harness](https://testing.adcontextprotocol.org) to validate your implementation's compliance with the AdCP specification. This interactive tool allows you to test all AdCP tasks and verify correct behavior across different scenarios. +### Testing via Addie + +The easiest way to test your AdCP agent is to ask Addie in Slack: + +> "Hey Addie, test my sales agent at https://sales.example.com" + +Addie can run comprehensive E2E tests including: + +**Standard Scenarios:** +- **health_check** - Verify agent responds +- **discovery** - Test `get_products`, `list_creative_formats`, `list_authorized_properties` +- **create_media_buy** - Discovery + create a test campaign +- **full_sales_flow** - Complete lifecycle: create → update → delivery +- **creative_sync** - Test `sync_creatives` flow +- **creative_inline** - Test inline creatives in `create_media_buy` +- **pricing_models** - Analyze pricing options across channels + +**Edge Case Scenarios:** +- **error_handling** - Verify proper discriminated union error responses +- **validation** - Test rejection of invalid inputs (negative budgets, invalid enums) +- **pricing_edge_cases** - Test auction vs fixed pricing, min_spend requirements, bid_price handling +- **temporal_validation** - Test date/time ordering, ISO 8601 format validation + +**Behavioral Analysis:** +- **behavior_analysis** - Analyze agent characteristics: authentication requirements, brand_manifest requirements, brief relevance filtering, channel filtering behavior + +By default tests run in dry-run mode. For real testing, ask Addie to run without dry-run. + +### Sales Agent Compliance Checklist + +Use this checklist to verify your sales agent implementation covers all required features: + +**Core Discovery (Required)** +- [ ] `get_products` - Returns products with pricing_options, format_ids, delivery_type +- [ ] `list_creative_formats` - Returns supported formats and creative agents +- [ ] `list_authorized_properties` - Returns publisher domains (if applicable) + +**Media Buy Lifecycle (Required)** +- [ ] `create_media_buy` - Accepts packages with product_id, pricing_option_id, budget +- [ ] `update_media_buy` - Supports PATCH semantics for budget, pacing, targeting +- [ ] `get_media_buy_delivery` - Returns impressions, spend, status + +**Creative Management (Required for most channels)** +- [ ] `sync_creatives` - Upsert creatives with per-item action tracking +- [ ] `list_creatives` - Query creative library with filtering +- [ ] Support inline creatives in `create_media_buy` +- [ ] Support creative references (`creative_ids`) + +**Pricing Models (as applicable)** +- [ ] CPM - Cost per thousand impressions +- [ ] vCPM - Viewable CPM (MRC standard) +- [ ] CPCV - Cost per completed view +- [ ] CPC - Cost per click +- [ ] CPP - Cost per rating point (TV/radio) +- [ ] Flat rate - Fixed cost sponsorships +- [ ] Auction pricing - Support bid_price when is_fixed=false + +**Creative Types** +- [ ] Static creatives (image, video assets) +- [ ] Reference creatives (creative_ids to existing library) +- [ ] Generative creatives (manifest-based) +- [ ] Parameterized creatives (with substitution) + +**Response Patterns** +- [ ] Discriminated union responses (success XOR errors) +- [ ] Schema-compliant responses (validate against JSON schemas) +- [ ] Async operations return status: submitted/working/completed +- [ ] Per-item errors in batch operations (e.g., sync_creatives) + +**Testing Support** +- [ ] `X-Dry-Run` header support +- [ ] `X-Test-Session-ID` for parallel test isolation +- [ ] `X-Mock-Time` for time simulation + +**Edge Case Validation (Required)** +- [ ] Reject negative budget values +- [ ] Reject invalid pacing enum values +- [ ] Reject end_time before start_time +- [ ] Reject invalid ISO 8601 date formats +- [ ] Return proper error for non-existent product_id +- [ ] Require bid_price for auction pricing options +- [ ] Reject budget below min_spend_per_package +- [ ] Reject creative weight > 100 +- [ ] Return discriminated union error responses (success XOR errors, never both) ## Testing Modes From e81745ebf82101b8c5feb61518b1d0f64637f7d8 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 3 Jan 2026 15:06:34 -0500 Subject: [PATCH 4/8] Add GitHub issue offer for open-source agent test failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When users test agents via Addie and the tests fail on a known open-source agent (test-agent.adcontextprotocol.org, wonderstruck.sales-agent.scope3.com, or creative.adcontextprotocol.org), Addie now offers to help file a GitHub issue to the appropriate repository. Also refactors to use @adcp/client/testing library directly instead of local agent-tester.ts wrapper: - Imports runAgentTests, formatTestResults, createTestClient from library - Adds call_adcp_tool handler for raw task execution - Removes redundant local agent-tester.ts (2570 lines) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .changeset/github-issue-offer.md | 4 + package-lock.json | 8 +- package.json | 2 +- server/src/addie/mcp/agent-tester.ts | 2570 -------------------------- server/src/addie/mcp/member-tools.ts | 183 +- server/tsconfig.json | 2 +- 6 files changed, 191 insertions(+), 2578 deletions(-) create mode 100644 .changeset/github-issue-offer.md delete mode 100644 server/src/addie/mcp/agent-tester.ts diff --git a/.changeset/github-issue-offer.md b/.changeset/github-issue-offer.md new file mode 100644 index 0000000000..a780d5e737 --- /dev/null +++ b/.changeset/github-issue-offer.md @@ -0,0 +1,4 @@ +--- +--- + +Add GitHub issue offer for open-source agent test failures diff --git a/package-lock.json b/package-lock.json index c249c1f946..9c10e5d663 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "adcontextprotocol", "version": "2.5.1", "dependencies": { - "@adcp/client": "^3.4.0", + "@adcp/client": "^3.5.0", "@anthropic-ai/sdk": "^0.71.2", "@modelcontextprotocol/sdk": "^1.24.3", "@mozilla/readability": "^0.6.0", @@ -86,9 +86,9 @@ } }, "node_modules/@adcp/client": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/@adcp/client/-/client-3.4.0.tgz", - "integrity": "sha512-DRC/sib4y05Slg3KP8um2cpxc4GRKtKTub7/hQMuX4CKNB76PxEdGed/Lubiq2CoaBRnygHBIeo0gsDX55tcdA==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@adcp/client/-/client-3.5.0.tgz", + "integrity": "sha512-pcCUMIztHY3tVlTOecw6rh8vUmKUyZ1VJ0DNDX17LuvqnGhhFvwmoTbUlhvn47pWlSFu5vhK7TBGFVeO0p0drQ==", "license": "MIT", "dependencies": { "better-sqlite3": "^12.4.1", diff --git a/package.json b/package.json index 92210d26bc..c334c3aad9 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "verify-version-sync": "node scripts/verify-version-sync.js" }, "dependencies": { - "@adcp/client": "^3.4.0", + "@adcp/client": "^3.5.0", "@anthropic-ai/sdk": "^0.71.2", "@modelcontextprotocol/sdk": "^1.24.3", "@mozilla/readability": "^0.6.0", diff --git a/server/src/addie/mcp/agent-tester.ts b/server/src/addie/mcp/agent-tester.ts deleted file mode 100644 index 2f77b307b4..0000000000 --- a/server/src/addie/mcp/agent-tester.ts +++ /dev/null @@ -1,2570 +0,0 @@ -/** - * AdCP Agent E2E Tester - * - * Provides comprehensive end-to-end testing of AdCP agents (sales, creative, signals). - * Replaces the testing.adcontextprotocol.org harness with conversational testing via Addie. - * - * Features: - * - Channel-aware testing (only tests features the agent supports) - * - Optional dry-run mode (real testing requires actual media buys) - * - Comprehensive scenario coverage based on AdCP spec - * - Schema validation via @adcp/client - */ - -import { AdCPClient } from '@adcp/client'; -import { logger } from '../../logger.js'; - -// Generic task result from executeTask - we use any for data since responses vary by task -interface TaskResult { - success: boolean; - data?: any; - error?: string; -} - -// Test scenarios that can be run -export type TestScenario = - | 'health_check' // Just check if agent responds - | 'discovery' // get_products, list_creative_formats, list_authorized_properties - | 'create_media_buy' // Discovery + create a test media buy - | 'full_sales_flow' // Full lifecycle: discovery -> create -> update -> delivery - | 'creative_sync' // Test sync_creatives flow - | 'creative_inline' // Test inline creatives in create_media_buy - | 'creative_reference' // Test reference creatives (creative_ids) - | 'pricing_models' // Test different pricing models the agent supports - | 'creative_flow' // Creative agent: list_formats -> build -> preview - | 'signals_flow' // Signals agent: get_signals -> activate - // Edge case testing scenarios - | 'error_handling' // Test agent returns proper error responses - | 'validation' // Test schema validation (invalid inputs should be rejected) - | 'pricing_edge_cases' // Test auction vs fixed pricing, min spend, bid_price requirements - | 'temporal_validation' // Test date/time ordering and format validation - // Behavioral analysis scenarios - | 'behavior_analysis' // Analyze agent behavior: auth requirements, brief relevance, filtering - // Response consistency scenarios - | 'response_consistency'; // Check for schema errors, pagination bugs, data mismatches - -export interface TestOptions { - // Custom brief for product discovery - brief?: string; - // Budget for test media buy (default: 1000) - budget?: number; - // Specific format IDs to test - format_ids?: string[]; - // Test session ID for isolation - test_session_id?: string; - // Whether to use dry-run mode (default: true for safety) - dry_run?: boolean; - // Channels to focus on (if not specified, tests all agent supports) - channels?: string[]; - // Specific pricing models to test - pricing_models?: string[]; - // Authentication for agents that require it - auth?: { - type: 'bearer'; - token: string; - }; -} - -export interface TestStepResult { - step: string; - task?: string; - passed: boolean; - duration_ms: number; - details?: string; - error?: string; - response_preview?: string; - // For tracking what was created (for cleanup or follow-up) - created_id?: string; -} - -export interface AgentProfile { - name: string; - tools: string[]; - channels?: string[]; - pricing_models?: string[]; - format_ids?: string[]; - delivery_types?: string[]; -} - -export interface TestResult { - agent_url: string; - scenario: TestScenario; - overall_passed: boolean; - steps: TestStepResult[]; - summary: string; - total_duration_ms: number; - tested_at: string; - // Agent profile discovered during testing - agent_profile?: AgentProfile; - // Was this run in dry-run mode? - dry_run: boolean; -} - -/** - * Create a test client for an agent - */ -function createTestClient( - agentUrl: string, - protocol: 'mcp' | 'a2a' = 'mcp', - options: TestOptions = {} -) { - const headers: Record = {}; - - // Dry-run is true by default for safety - if (options.dry_run !== false) { - headers['X-Dry-Run'] = 'true'; - } - - if (options.test_session_id) { - headers['X-Test-Session-ID'] = options.test_session_id; - } - - // Build agent config with auth_token if provided - const agentConfig: { - id: string; - name: string; - agent_uri: string; - protocol: 'mcp' | 'a2a'; - auth_token?: string; - } = { - id: 'test', - name: 'E2E Test Client', - agent_uri: agentUrl, - protocol, - }; - - // Add auth_token to agent config - the library will use it automatically - if (options.auth?.type === 'bearer' && options.auth?.token) { - agentConfig.auth_token = options.auth.token; - } - - const multiClient = new AdCPClient([agentConfig], { - headers, - }); - - return multiClient.agent('test'); -} - -/** - * Run a single test step with timing - */ -async function runStep( - stepName: string, - taskName: string | undefined, - fn: () => Promise -): Promise<{ result?: T; step: TestStepResult }> { - const start = Date.now(); - try { - const result = await fn(); - const duration = Date.now() - start; - return { - result, - step: { - step: stepName, - task: taskName, - passed: true, - duration_ms: duration, - }, - }; - } catch (error) { - const duration = Date.now() - start; - const errorMessage = error instanceof Error ? error.message : String(error); - return { - step: { - step: stepName, - task: taskName, - passed: false, - duration_ms: duration, - error: errorMessage, - }, - }; - } -} - -/** - * Discover agent profile - what capabilities does this agent have? - */ -async function discoverAgentProfile( - client: ReturnType -): Promise<{ profile: AgentProfile; step: TestStepResult }> { - const { result: agentInfo, step } = await runStep( - 'Discover agent capabilities', - 'getAgentInfo', - () => client.getAgentInfo() - ); - - const profile: AgentProfile = { - name: agentInfo?.name || 'Unknown', - tools: agentInfo?.tools?.map((t: { name: string }) => t.name) || [], - }; - - if (agentInfo) { - step.details = `Agent: ${profile.name}, Tools: ${profile.tools.length}`; - step.response_preview = JSON.stringify({ - name: profile.name, - tools: profile.tools, - }, null, 2); - } - - return { profile, step }; -} - -/** - * Discover what channels, pricing models, formats the agent supports - * by calling get_products and analyzing the response - */ -async function discoverAgentCapabilities( - client: ReturnType, - profile: AgentProfile, - options: TestOptions -): Promise<{ capabilities: Partial; steps: TestStepResult[] }> { - const steps: TestStepResult[] = []; - const capabilities: Partial = {}; - - if (!profile.tools.includes('get_products')) { - return { capabilities, steps }; - } - - const brief = options.brief || 'Show me all available advertising products across all channels'; - // Include brand_manifest as some agents require it (e.g., tenant-specific agents) - const getProductsParams: Record = { - brief, - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - }, - }; - const { result, step } = await runStep( - 'Discover products for capability analysis', - 'get_products', - async () => client.executeTask('get_products', getProductsParams) as Promise - ); - - if (result?.success && result?.data?.products) { - const products = result.data.products as any[]; - - // Extract unique channels - const channels = new Set(); - const pricingModels = new Set(); - const formatIds = new Set(); - const deliveryTypes = new Set(); - - for (const product of products) { - // Channels from product - if (product.channels) { - for (const ch of product.channels) { - channels.add(ch); - } - } - // Delivery type - if (product.delivery_type) { - deliveryTypes.add(product.delivery_type); - } - // Pricing models - if (product.pricing_options) { - for (const po of product.pricing_options) { - if (po.model) pricingModels.add(po.model); - } - } - // Format IDs - if (product.format_ids) { - for (const fid of product.format_ids) { - const id = typeof fid === 'string' ? fid : fid.id; - if (id) formatIds.add(id); - } - } - } - - capabilities.channels = Array.from(channels); - capabilities.pricing_models = Array.from(pricingModels); - capabilities.format_ids = Array.from(formatIds); - capabilities.delivery_types = Array.from(deliveryTypes); - - step.details = `Found ${products.length} products across ${channels.size} channel(s), ${pricingModels.size} pricing model(s)`; - step.response_preview = JSON.stringify({ - products_count: products.length, - channels: capabilities.channels, - pricing_models: capabilities.pricing_models, - delivery_types: capabilities.delivery_types, - format_count: capabilities.format_ids?.length, - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'get_products failed'; - } - - steps.push(step); - return { capabilities, steps }; -} - -/** - * Test: Health Check - * Verifies the agent is responding and has an agent card - */ -async function testHealthCheck(agentUrl: string, options: TestOptions): Promise { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { step } = await discoverAgentProfile(client); - steps.push(step); - - return steps; -} - -/** - * Test: Discovery - * Tests product discovery, format listing, and property listing - */ -async function testDiscovery(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // Discover agent profile - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // Discover capabilities - const { capabilities, steps: capSteps } = await discoverAgentCapabilities(client, profile, options); - steps.push(...capSteps); - - // Merge capabilities into profile - Object.assign(profile, capabilities); - - // List creative formats (if available) - if (profile.tools.includes('list_creative_formats')) { - const { result, step } = await runStep( - 'List creative formats', - 'list_creative_formats', - async () => client.executeTask('list_creative_formats', {}) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const formatCount = data.format_ids?.length || data.formats?.length || 0; - const creativeAgents = data.creative_agents || []; - step.details = `Found ${formatCount} format(s), ${creativeAgents.length} creative agent(s)`; - step.response_preview = JSON.stringify({ - format_ids: (data.format_ids || data.formats?.map((f: any) => f.format_id))?.slice(0, 5), - creative_agents: creativeAgents.map((a: any) => a.agent_url || a.url), - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'list_creative_formats returned unsuccessful result'; - } - steps.push(step); - } - - // List authorized properties (if available) - if (profile.tools.includes('list_authorized_properties')) { - const { result, step } = await runStep( - 'List authorized properties', - 'list_authorized_properties', - async () => client.executeTask('list_authorized_properties', {}) as Promise - ); - - const properties = result?.data?.authorized_properties as any[] | undefined; - if (result?.success && properties) { - step.details = `Found ${properties.length} authorized propert(ies)`; - step.response_preview = JSON.stringify({ - properties_count: properties.length, - domains: properties.slice(0, 3).map((p: any) => p.domain), - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'list_authorized_properties returned unsuccessful result'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Find a suitable product for testing based on options - */ -function selectProduct(products: any[], options: TestOptions): any | null { - // If channels specified, filter to matching products - let candidates = products; - - if (options.channels?.length) { - candidates = products.filter(p => - p.channels?.some((ch: string) => options.channels!.includes(ch)) - ); - } - - // If pricing models specified, filter further - if (options.pricing_models?.length) { - candidates = candidates.filter(p => - p.pricing_options?.some((po: any) => - options.pricing_models!.includes(po.model) - ) - ); - } - - // Return first matching or first product - return candidates[0] || products[0] || null; -} - -/** - * Select a pricing option from a product - */ -function selectPricingOption(product: any, preferredModels?: string[]): any | null { - const options = product.pricing_options || []; - - if (preferredModels?.length) { - const preferred = options.find((po: any) => preferredModels.includes(po.model)); - if (preferred) return preferred; - } - - return options[0] || null; -} - -/** - * Build a create_media_buy request - */ -function buildCreateMediaBuyRequest( - product: any, - pricingOption: any, - options: TestOptions, - extras: { - inline_creatives?: any[]; - creative_ids?: string[]; - } = {} -): any { - const budget = options.budget || 1000; - const now = new Date(); - const startTime = new Date(now.getTime() + 24 * 60 * 60 * 1000); // Tomorrow - const endTime = new Date(startTime.getTime() + 7 * 24 * 60 * 60 * 1000); // 7 days later - - const isAuction = pricingOption.model === 'auction' || - pricingOption.is_fixed === false || - pricingOption.floor_price !== undefined; - - const packageRequest: any = { - buyer_ref: `pkg-test-${Date.now()}`, - product_id: product.product_id, - budget, - pricing_option_id: pricingOption.pricing_option_id, - }; - - // Add bid_price if auction-based - if (isAuction && pricingOption.floor_price) { - packageRequest.bid_price = pricingOption.floor_price * 1.5; - } - - // Add inline creatives if provided - if (extras.inline_creatives?.length) { - packageRequest.creatives = extras.inline_creatives; - } - - // Add creative references if provided - if (extras.creative_ids?.length) { - packageRequest.creative_ids = extras.creative_ids; - } - - return { - buyer_ref: `e2e-test-${Date.now()}`, - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - }, - start_time: startTime.toISOString(), - end_time: endTime.toISOString(), - packages: [packageRequest], - }; -} - -/** - * Test: Create Media Buy - * Discovers products, then creates a test media buy - */ -async function testCreateMediaBuy(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile; mediaBuyId?: string }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // First run discovery - const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); - steps.push(...discoverySteps); - - if (!profile?.tools.includes('create_media_buy')) { - steps.push({ - step: 'Create media buy', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: 'Agent does not support create_media_buy', - }); - return { steps, profile }; - } - - // Get products - const { result: productsResult } = await runStep( - 'Fetch products for media buy', - 'get_products', - async () => client.executeTask('get_products', { - brief: options.brief || 'Looking for display advertising products', - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - }, - }) as Promise - ); - - const products = productsResult?.data?.products as any[] | undefined; - if (!productsResult?.success || !products?.length) { - steps.push({ - step: 'Create media buy', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: 'No products available to create media buy', - }); - return { steps, profile }; - } - - const product = selectProduct(products, options); - const pricingOption = selectPricingOption(product, options.pricing_models); - - if (!pricingOption) { - steps.push({ - step: 'Create media buy', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: `Product "${product.name}" has no pricing options`, - }); - return { steps, profile }; - } - - const createRequest = buildCreateMediaBuyRequest(product, pricingOption, options); - - // Create the media buy - const { result: createResult, step: createStep } = await runStep( - 'Create media buy', - 'create_media_buy', - async () => client.executeTask('create_media_buy', createRequest) as Promise - ); - - let mediaBuyId: string | undefined; - - if (createResult?.success && createResult?.data) { - const mediaBuy = createResult.data as any; - mediaBuyId = mediaBuy.media_buy_id || mediaBuy.media_buy?.media_buy_id; - const status = mediaBuy.status || mediaBuy.media_buy?.status; - const packages = mediaBuy.packages || mediaBuy.media_buy?.packages; - createStep.details = `Created media buy: ${mediaBuyId}, status: ${status}`; - createStep.created_id = mediaBuyId; - createStep.response_preview = JSON.stringify({ - media_buy_id: mediaBuyId, - status, - packages_count: packages?.length, - pricing_model: pricingOption.model, - product_name: product.name, - }, null, 2); - } else if (createResult && !createResult.success) { - createStep.passed = false; - createStep.error = createResult.error || 'create_media_buy returned unsuccessful result'; - } - steps.push(createStep); - - return { steps, profile, mediaBuyId }; -} - -/** - * Test: Full Sales Flow - * Complete lifecycle: discovery -> create -> update -> delivery - */ -async function testFullSalesFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // Run create media buy flow first - const { steps: createSteps, profile, mediaBuyId } = await testCreateMediaBuy(agentUrl, options); - steps.push(...createSteps); - - if (!mediaBuyId) { - return { steps, profile }; - } - - // Test update_media_buy if available - if (profile?.tools.includes('update_media_buy')) { - const { result: updateResult, step: updateStep } = await runStep( - 'Update media buy (increase budget)', - 'update_media_buy', - async () => client.executeTask('update_media_buy', { - media_buy_id: mediaBuyId, - packages: [{ - package_id: 'pkg-0', - budget: (options.budget || 1000) * 1.5, - }], - }) as Promise - ); - - if (updateResult?.success && updateResult?.data) { - const data = updateResult.data as any; - const status = data.status || data.media_buy?.status; - updateStep.details = `Updated media buy, status: ${status}`; - updateStep.response_preview = JSON.stringify({ - media_buy_id: data.media_buy_id || data.media_buy?.media_buy_id, - status, - }, null, 2); - } else if (updateResult && !updateResult.success) { - updateStep.passed = false; - updateStep.error = updateResult.error || 'update_media_buy returned unsuccessful result'; - } - steps.push(updateStep); - } - - // Test get_media_buy_delivery if available - if (profile?.tools.includes('get_media_buy_delivery')) { - const { result: deliveryResult, step: deliveryStep } = await runStep( - 'Get delivery metrics', - 'get_media_buy_delivery', - async () => client.executeTask('get_media_buy_delivery', { - media_buy_ids: [mediaBuyId], - }) as Promise - ); - - if (deliveryResult?.success && deliveryResult?.data) { - const delivery = deliveryResult.data as any; - deliveryStep.details = `Retrieved delivery metrics`; - deliveryStep.response_preview = JSON.stringify({ - has_deliveries: !!(delivery.deliveries?.length || delivery.media_buys?.length), - }, null, 2); - } else if (deliveryResult && !deliveryResult.success) { - deliveryStep.passed = false; - deliveryStep.error = deliveryResult.error || 'get_media_buy_delivery returned unsuccessful result'; - } - steps.push(deliveryStep); - } - - return { steps, profile }; -} - -/** - * Test: Creative Sync Flow - * Tests sync_creatives separately from create_media_buy - */ -async function testCreativeSync(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // Discover profile - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profile.tools.includes('sync_creatives')) { - steps.push({ - step: 'Sync creatives', - task: 'sync_creatives', - passed: false, - duration_ms: 0, - error: 'Agent does not support sync_creatives', - }); - return { steps, profile }; - } - - // Get format info first - let formatId = 'display_300x250'; // Default - if (profile.tools.includes('list_creative_formats')) { - const { result: formatsResult } = await runStep( - 'Get formats for creative', - 'list_creative_formats', - async () => client.executeTask('list_creative_formats', {}) as Promise - ); - - if (formatsResult?.success && formatsResult?.data) { - const data = formatsResult.data as any; - const firstFormat = data.format_ids?.[0] || data.formats?.[0]; - if (firstFormat) { - formatId = typeof firstFormat === 'string' ? firstFormat : (firstFormat.id || firstFormat.format_id); - } - } - } - - // Test sync_creatives with a simple creative - // Assets must be an object keyed by asset_role, not an array - const testCreative = { - creative_id: `test-creative-${Date.now()}`, - name: 'E2E Test Creative', - format_id: formatId, - assets: { - primary: { - url: 'https://via.placeholder.com/300x250', - width: 300, - height: 250, - format: 'png', - }, - }, - }; - - const { result: syncResult, step: syncStep } = await runStep( - 'Sync creative to library', - 'sync_creatives', - async () => client.executeTask('sync_creatives', { - creatives: [testCreative], - }) as Promise - ); - - if (syncResult?.success && syncResult?.data) { - const data = syncResult.data as any; - const creatives = data.creatives || []; - const actions = creatives.map((c: any) => c.action); - syncStep.details = `Synced ${creatives.length} creative(s), actions: ${actions.join(', ')}`; - syncStep.response_preview = JSON.stringify({ - creatives_count: creatives.length, - actions: actions, - creative_ids: creatives.map((c: any) => c.creative_id), - }, null, 2); - } else if (syncResult && !syncResult.success) { - syncStep.passed = false; - syncStep.error = syncResult.error || 'sync_creatives returned unsuccessful result'; - } - steps.push(syncStep); - - // Test list_creatives if available - if (profile.tools.includes('list_creatives')) { - const { result: listResult, step: listStep } = await runStep( - 'List creatives in library', - 'list_creatives', - async () => client.executeTask('list_creatives', {}) as Promise - ); - - if (listResult?.success && listResult?.data) { - const data = listResult.data as any; - const creatives = data.creatives || []; - const querySummary = data.query_summary; - const totalMatching = querySummary?.total_matching; - const returned = querySummary?.returned ?? creatives.length; - - // Check for pagination bug: total_matching > 0 but returned = 0 - if (totalMatching !== undefined && totalMatching > 0 && returned === 0) { - listStep.passed = false; - listStep.error = `Pagination bug: query_summary shows ${totalMatching} total_matching but returned ${returned} creatives`; - listStep.response_preview = JSON.stringify({ - total_matching: totalMatching, - returned, - creatives_count: creatives.length, - pagination: data.pagination, - }, null, 2); - } else { - listStep.details = `Found ${creatives.length} creative(s) in library`; - listStep.response_preview = JSON.stringify({ - creatives_count: creatives.length, - total_matching: totalMatching, - statuses: Array.from(new Set(creatives.map((c: any) => c.status))), - }, null, 2); - } - } else if (listResult && !listResult.success) { - listStep.passed = false; - listStep.error = listResult.error || 'list_creatives returned unsuccessful result'; - } - steps.push(listStep); - } - - return { steps, profile }; -} - -/** - * Test: Creative Inline Flow - * Tests providing creatives inline in create_media_buy - */ -async function testCreativeInline(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // Discovery first - const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); - steps.push(...discoverySteps); - - if (!profile?.tools.includes('create_media_buy')) { - steps.push({ - step: 'Create media buy with inline creatives', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: 'Agent does not support create_media_buy', - }); - return { steps, profile }; - } - - // Get products - const { result: productsResult } = await runStep( - 'Fetch products', - 'get_products', - async () => client.executeTask('get_products', { - brief: options.brief || 'Looking for display advertising products', - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - }, - }) as Promise - ); - - const products = productsResult?.data?.products as any[] | undefined; - if (!products?.length) { - steps.push({ - step: 'Create media buy with inline creatives', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: 'No products available', - }); - return { steps, profile }; - } - - const product = selectProduct(products, options); - const pricingOption = selectPricingOption(product, options.pricing_models); - - if (!pricingOption) { - steps.push({ - step: 'Create media buy with inline creatives', - task: 'create_media_buy', - passed: false, - duration_ms: 0, - error: 'No pricing options available', - }); - return { steps, profile }; - } - - // Build inline creative - const formatId = product.format_ids?.[0]; - const formatIdValue = typeof formatId === 'string' ? formatId : (formatId?.id || 'display_300x250'); - - // Assets must be an object keyed by asset_role, not an array - const inlineCreative = { - creative_id: `inline-creative-${Date.now()}`, - name: 'Inline Test Creative', - format_id: formatIdValue, - assets: { - primary: { - url: 'https://via.placeholder.com/300x250', - width: 300, - height: 250, - format: 'png', - }, - }, - }; - - const createRequest = buildCreateMediaBuyRequest(product, pricingOption, options, { - inline_creatives: [inlineCreative], - }); - - const { result: createResult, step: createStep } = await runStep( - 'Create media buy with inline creatives', - 'create_media_buy', - async () => client.executeTask('create_media_buy', createRequest) as Promise - ); - - if (createResult?.success && createResult?.data) { - const mediaBuy = createResult.data as any; - const mediaBuyId = mediaBuy.media_buy_id || mediaBuy.media_buy?.media_buy_id; - createStep.details = `Created media buy with inline creative: ${mediaBuyId}`; - createStep.response_preview = JSON.stringify({ - media_buy_id: mediaBuyId, - status: mediaBuy.status || mediaBuy.media_buy?.status, - inline_creative_used: true, - }, null, 2); - } else if (createResult && !createResult.success) { - createStep.passed = false; - createStep.error = createResult.error || 'create_media_buy returned unsuccessful result'; - } - steps.push(createStep); - - return { steps, profile }; -} - -/** - * Test: Pricing Models - * Tests different pricing models the agent supports - */ -async function testPricingModels(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // Discovery first - const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); - steps.push(...discoverySteps); - - if (!profile?.pricing_models?.length) { - steps.push({ - step: 'Test pricing models', - passed: false, - duration_ms: 0, - error: 'No pricing models discovered', - }); - return { steps, profile }; - } - - // Get products to analyze pricing - const { result: productsResult } = await runStep( - 'Fetch products for pricing analysis', - 'get_products', - async () => client.executeTask('get_products', { - brief: 'Show all products', - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - }, - }) as Promise - ); - - const products = productsResult?.data?.products as any[] | undefined; - if (!products?.length) { - return { steps, profile }; - } - - // Analyze pricing model distribution - const pricingAnalysis: Record = {}; - - for (const product of products) { - for (const po of (product.pricing_options || [])) { - const model = po.model || 'unknown'; - if (!pricingAnalysis[model]) { - pricingAnalysis[model] = { count: 0, fixed: 0, auction: 0 }; - } - pricingAnalysis[model].count++; - if (po.is_fixed === false || po.floor_price !== undefined) { - pricingAnalysis[model].auction++; - } else { - pricingAnalysis[model].fixed++; - } - } - } - - steps.push({ - step: 'Analyze pricing models', - passed: true, - duration_ms: 0, - details: `Found ${Object.keys(pricingAnalysis).length} pricing model(s)`, - response_preview: JSON.stringify(pricingAnalysis, null, 2), - }); - - return { steps, profile }; -} - -/** - * Test: Creative Flow (for creative agents) - */ -async function testCreativeFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // List creative formats - if (profile.tools.includes('list_creative_formats')) { - const { result, step } = await runStep( - 'List creative formats', - 'list_creative_formats', - async () => client.executeTask('list_creative_formats', {}) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const formats = data.formats || []; - step.details = `Found ${formats.length} format(s)`; - step.response_preview = JSON.stringify({ - formats_count: formats.length, - format_names: formats.slice(0, 5).map((f: any) => f.name || f.format_id), - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'list_creative_formats failed'; - } - steps.push(step); - } - - // Build creative (if available) - if (profile.tools.includes('build_creative')) { - const { result, step } = await runStep( - 'Build creative', - 'build_creative', - async () => client.executeTask('build_creative', { - format_id: options.format_ids?.[0] || 'display_300x250', - brand_manifest: { - name: 'E2E Test Brand', - url: 'https://test.example.com', - tagline: 'Testing the future of advertising', - }, - prompt: 'Create a simple display ad for a tech product', - }) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - step.details = `Built creative successfully`; - step.response_preview = JSON.stringify({ - creative_id: data.creative_id || data.creative?.creative_id, - format_id: data.format_id || data.creative?.format_id, - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'build_creative failed'; - } - steps.push(step); - } - - // Preview creative (if available) - if (profile.tools.includes('preview_creative')) { - const { result, step } = await runStep( - 'Preview creative', - 'preview_creative', - async () => client.executeTask('preview_creative', { - creative: { - format_id: options.format_ids?.[0] || 'display_300x250', - name: 'Test Creative', - assets: [], - }, - }) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - step.details = `Generated preview`; - step.response_preview = JSON.stringify({ - has_renders: !!(data.renders?.length || data.preview_url), - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'preview_creative failed'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Test: Signals Flow (for signals agents) - */ -async function testSignalsFlow(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // Get signals - if (profile.tools.includes('get_signals')) { - const { result, step } = await runStep( - 'Get signals', - 'get_signals', - async () => client.executeTask('get_signals', { - brief: 'Looking for audience segments interested in technology', - }) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const signals = data.signals || []; - step.details = `Found ${signals.length} signal(s)`; - step.response_preview = JSON.stringify({ - signals_count: signals.length, - signal_names: signals.slice(0, 3).map((s: any) => s.name), - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'get_signals failed'; - } - steps.push(step); - } - - // Activate signal (if available) - if (profile.tools.includes('activate_signal')) { - const { result, step } = await runStep( - 'Activate signal', - 'activate_signal', - async () => client.executeTask('activate_signal', { - signal_id: 'test-signal-id', - destination: { - platform: 'test-platform', - account_id: 'test-account', - }, - }) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - step.details = `Signal activation submitted`; - step.response_preview = JSON.stringify({ - status: data.status || data.deployment?.status, - }, null, 2); - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'activate_signal failed'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Test: Error Handling - * Verifies the agent returns proper discriminated union error responses - */ -async function testErrorHandling(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // Test 1: Invalid product_id in create_media_buy should return proper error - if (profile.tools.includes('create_media_buy')) { - const { result, step } = await runStep( - 'Invalid product_id error response', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `error-test-${Date.now()}`, - brand_manifest: { - name: 'Error Test Brand', - url: 'https://test.example.com', - }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-error-test', - product_id: 'NONEXISTENT_PRODUCT_ID_12345', - budget: 1000, - pricing_option_id: 'nonexistent-pricing', - }], - }) as Promise - ); - - // For error handling test, we EXPECT an error - passing means the error was returned properly - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent correctly returned error for invalid product_id'; - step.response_preview = JSON.stringify({ error: result.error }, null, 2); - } else if (result?.success) { - step.passed = false; - step.error = 'Agent accepted invalid product_id - should have returned error'; - } else { - step.passed = false; - step.error = 'Agent returned neither success nor proper error response'; - } - steps.push(step); - } - - // Test 2: Missing required field in get_products (brief is often required) - if (profile.tools.includes('get_products')) { - const { result, step } = await runStep( - 'Empty request handling', - 'get_products', - async () => client.executeTask('get_products', {}) as Promise - ); - - // Some agents may accept empty requests, others may require brief - if (result?.success) { - step.passed = true; - step.details = 'Agent accepts empty get_products request (permissive)'; - } else if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent requires brief/brand_manifest (stricter validation)'; - step.response_preview = JSON.stringify({ error: result.error }, null, 2); - } else { - step.passed = false; - step.error = 'Unclear response - neither success nor proper error'; - } - steps.push(step); - } - - // Test 3: Invalid format_id in sync_creatives - if (profile.tools.includes('sync_creatives')) { - const { result, step } = await runStep( - 'Invalid format_id error response', - 'sync_creatives', - async () => client.executeTask('sync_creatives', { - creatives: [{ - creative_id: `invalid-format-test-${Date.now()}`, - name: 'Invalid Format Test', - format_id: 'TOTALLY_INVALID_FORMAT_ID_999', - assets: { - primary: { - url: 'https://via.placeholder.com/300x250', - width: 300, - height: 250, - format: 'png', - }, - }, - }], - }) as Promise - ); - - // Expect error for invalid format_id - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent correctly rejected invalid format_id'; - step.response_preview = JSON.stringify({ error: result.error }, null, 2); - } else if (result?.success) { - // Some agents may be permissive and accept any format_id - step.passed = true; - step.details = 'Agent accepts unknown format_ids (permissive mode)'; - } else { - step.passed = false; - step.error = 'Unclear response for invalid format_id'; - } - steps.push(step); - } - - // Test 4: get_media_buy_delivery with non-existent media_buy_id - if (profile.tools.includes('get_media_buy_delivery')) { - const { result, step } = await runStep( - 'Non-existent media_buy_id error', - 'get_media_buy_delivery', - async () => client.executeTask('get_media_buy_delivery', { - media_buy_ids: ['NONEXISTENT_MEDIA_BUY_ID_99999'], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent correctly returned error for non-existent media buy'; - step.response_preview = JSON.stringify({ error: result.error }, null, 2); - } else if (result?.success) { - // Check if it returned empty deliveries (also acceptable) - const data = result.data as any; - const deliveries = data?.deliveries || data?.media_buys || []; - if (deliveries.length === 0) { - step.passed = true; - step.details = 'Agent returned empty deliveries for non-existent media buy'; - } else { - step.passed = false; - step.error = 'Agent returned deliveries for non-existent media_buy_id'; - } - } else { - step.passed = false; - step.error = 'Unclear response for non-existent media_buy_id'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Test: Validation - * Tests that agents properly validate inputs and reject malformed requests - */ -async function testValidation(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // Test 1: Invalid enum value for pacing - if (profile.tools.includes('create_media_buy')) { - const { result, step } = await runStep( - 'Invalid pacing enum value', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `validation-test-${Date.now()}`, - brand_manifest: { name: 'Validation Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-validation', - product_id: 'test-product', - budget: 1000, - pricing_option_id: 'test-pricing', - pacing: 'INVALID_PACING_VALUE' as any, // Invalid - should be even/asap/front_loaded - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent rejected invalid pacing enum value'; - } else if (result?.success) { - step.passed = false; - step.error = 'Agent accepted invalid pacing value - should validate enums'; - } else { - step.passed = false; - step.error = 'Unclear validation response'; - } - steps.push(step); - } - - // Test 2: Zero budget package - if (profile.tools.includes('create_media_buy')) { - const { result, step } = await runStep( - 'Zero budget validation', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `zero-budget-test-${Date.now()}`, - brand_manifest: { name: 'Zero Budget Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-zero-budget', - product_id: 'test-product', - budget: 0, // Zero budget - should be rejected or flagged - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent rejected zero budget (strict validation)'; - } else if (result?.success) { - step.passed = true; - step.details = 'Agent accepts zero budget (permissive - schema allows minimum: 0)'; - } else { - step.passed = false; - step.error = 'Unclear response for zero budget'; - } - steps.push(step); - } - - // Test 3: Negative budget (definitely invalid) - if (profile.tools.includes('create_media_buy')) { - const { result, step } = await runStep( - 'Negative budget rejection', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `negative-budget-test-${Date.now()}`, - brand_manifest: { name: 'Negative Budget Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-negative', - product_id: 'test-product', - budget: -500, // Negative budget - MUST be rejected - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent correctly rejected negative budget'; - } else if (result?.success) { - step.passed = false; - step.error = 'CRITICAL: Agent accepted negative budget - must validate minimum: 0'; - } else { - step.passed = false; - step.error = 'Unclear response for negative budget'; - } - steps.push(step); - } - - // Test 4: Invalid creative weight (> 100) - if (profile.tools.includes('sync_creatives')) { - const { result, step } = await runStep( - 'Invalid creative weight (> 100)', - 'sync_creatives', - async () => client.executeTask('sync_creatives', { - creatives: [{ - creative_id: `weight-test-${Date.now()}`, - name: 'Weight Test Creative', - format_id: 'display_300x250', - weight: 150, // Invalid - max is 100 - assets: { - primary: { - url: 'https://via.placeholder.com/300x250', - width: 300, - height: 250, - format: 'png', - }, - }, - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent rejected weight > 100'; - } else if (result?.success) { - step.passed = false; - step.error = 'Agent accepted weight > 100 - should validate maximum: 100'; - } else { - step.passed = false; - step.error = 'Unclear response for invalid weight'; - } - steps.push(step); - } - - // Test 5: Empty creatives array - if (profile.tools.includes('sync_creatives')) { - const { result, step } = await runStep( - 'Empty creatives array handling', - 'sync_creatives', - async () => client.executeTask('sync_creatives', { - creatives: [], // Empty array - should be rejected or return empty - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent rejected empty creatives array'; - } else if (result?.success) { - step.passed = true; - step.details = 'Agent accepts empty creatives array (returns empty result)'; - } else { - step.passed = false; - step.error = 'Unclear response for empty creatives array'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Test: Pricing Edge Cases - * Tests auction vs fixed pricing, min spend requirements, bid_price handling - */ -async function testPricingEdgeCases(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - // First get products to understand pricing options - const { steps: discoverySteps, profile } = await testDiscovery(agentUrl, options); - steps.push(...discoverySteps); - - if (!profile?.tools.includes('create_media_buy') || !profile?.tools.includes('get_products')) { - steps.push({ - step: 'Pricing edge cases', - passed: false, - duration_ms: 0, - error: 'Agent does not support create_media_buy or get_products', - }); - return { steps, profile }; - } - - // Get products to find actual pricing options - const { result: productsResult } = await runStep( - 'Fetch products for pricing analysis', - 'get_products', - async () => client.executeTask('get_products', { - brief: 'Show all products with pricing details', - brand_manifest: { name: 'Pricing Test', url: 'https://test.example.com' }, - }) as Promise - ); - - const products = productsResult?.data?.products as any[] | undefined; - if (!products?.length) { - steps.push({ - step: 'Pricing edge cases', - passed: false, - duration_ms: 0, - error: 'No products available for pricing tests', - }); - return { steps, profile }; - } - - // Analyze products for auction vs fixed pricing - const auctionProducts: any[] = []; - const fixedProducts: any[] = []; - const productsWithMinSpend: any[] = []; - - for (const product of products) { - for (const po of (product.pricing_options || [])) { - if (po.is_fixed === false || po.floor_price !== undefined || po.price_guidance !== undefined) { - auctionProducts.push({ product, pricingOption: po }); - } else if (po.rate !== undefined) { - fixedProducts.push({ product, pricingOption: po }); - } - if (po.min_spend_per_package !== undefined && po.min_spend_per_package > 0) { - productsWithMinSpend.push({ product, pricingOption: po, minSpend: po.min_spend_per_package }); - } - } - } - - steps.push({ - step: 'Analyze pricing options', - passed: true, - duration_ms: 0, - details: `Found ${fixedProducts.length} fixed, ${auctionProducts.length} auction, ${productsWithMinSpend.length} with min spend`, - response_preview: JSON.stringify({ - fixed_count: fixedProducts.length, - auction_count: auctionProducts.length, - min_spend_count: productsWithMinSpend.length, - }, null, 2), - }); - - // Test 1: Auction pricing without bid_price (should fail) - if (auctionProducts.length > 0) { - const { product, pricingOption } = auctionProducts[0]; - const { result, step } = await runStep( - 'Auction pricing without bid_price', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `auction-no-bid-${Date.now()}`, - brand_manifest: { name: 'Auction Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-auction-no-bid', - product_id: product.product_id, - budget: 5000, - pricing_option_id: pricingOption.pricing_option_id, - // Intentionally missing bid_price for auction pricing - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent correctly requires bid_price for auction pricing'; - } else if (result?.success) { - step.passed = false; - step.error = 'Agent accepted auction pricing without bid_price - should require it'; - } else { - step.passed = false; - step.error = 'Unclear response for missing bid_price'; - } - steps.push(step); - } - - // Test 2: Fixed pricing with bid_price (should be ignored or rejected) - if (fixedProducts.length > 0) { - const { product, pricingOption } = fixedProducts[0]; - const { result, step } = await runStep( - 'Fixed pricing with unnecessary bid_price', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `fixed-with-bid-${Date.now()}`, - brand_manifest: { name: 'Fixed Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-fixed-bid', - product_id: product.product_id, - budget: 5000, - pricing_option_id: pricingOption.pricing_option_id, - bid_price: 15.00, // Unnecessary for fixed pricing - }], - }) as Promise - ); - - if (result?.success) { - step.passed = true; - step.details = 'Agent ignores bid_price for fixed pricing (permissive)'; - } else if (result && !result.success && result.error) { - step.passed = true; - step.details = 'Agent rejects bid_price for fixed pricing (strict)'; - } else { - step.passed = false; - step.error = 'Unclear response for fixed pricing with bid_price'; - } - steps.push(step); - } - - // Test 3: Budget below min_spend_per_package - if (productsWithMinSpend.length > 0) { - const { product, pricingOption, minSpend } = productsWithMinSpend[0]; - const underBudget = minSpend * 0.5; // 50% of minimum - - const { result, step } = await runStep( - 'Budget below min_spend_per_package', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `under-min-spend-${Date.now()}`, - brand_manifest: { name: 'Min Spend Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-under-min', - product_id: product.product_id, - budget: underBudget, - pricing_option_id: pricingOption.pricing_option_id, - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = `Agent rejected budget ${underBudget} below min_spend ${minSpend}`; - } else if (result?.success) { - step.passed = false; - step.error = `Agent accepted budget ${underBudget} below min_spend ${minSpend}`; - } else { - step.passed = false; - step.error = 'Unclear response for under-min-spend budget'; - } - steps.push(step); - } - - // Test 4: Bid below floor price for auction - if (auctionProducts.length > 0) { - const { product, pricingOption } = auctionProducts[0]; - const floorPrice = pricingOption.floor_price || pricingOption.price_guidance?.floor || 0; - - if (floorPrice > 0) { - const belowFloor = floorPrice * 0.5; // 50% of floor - - const { result, step } = await runStep( - 'Bid below floor price', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `below-floor-${Date.now()}`, - brand_manifest: { name: 'Floor Test', url: 'https://test.example.com' }, - start_time: new Date(Date.now() + 86400000).toISOString(), - end_time: new Date(Date.now() + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-below-floor', - product_id: product.product_id, - budget: 5000, - pricing_option_id: pricingOption.pricing_option_id, - bid_price: belowFloor, - }], - }) as Promise - ); - - if (result && !result.success && result.error) { - step.passed = true; - step.details = `Agent rejected bid ${belowFloor} below floor ${floorPrice}`; - } else if (result?.success) { - // Some agents may accept and just not win auctions - step.passed = true; - step.details = `Agent accepts bid below floor (may not win auctions)`; - } else { - step.passed = false; - step.error = 'Unclear response for below-floor bid'; - } - steps.push(step); - } - } - - return { steps, profile }; -} - -/** - * Test: Behavior Analysis - * Analyzes interesting behavioral characteristics of the agent: - * - Does it require authentication for get_products? - * - Does it require brand_manifest to return products? - * - Are responses filtered based on the brief or is everything returned? - */ -async function testBehaviorAnalysis(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - - // Create authenticated client for comparison - const authClient = createTestClient(agentUrl, 'mcp', options); - const { profile, step: profileStep } = await discoverAgentProfile(authClient); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - if (!profile.tools.includes('get_products')) { - steps.push({ - step: 'Behavior analysis', - passed: false, - duration_ms: 0, - error: 'Agent does not support get_products - cannot analyze behavior', - }); - return { steps, profile }; - } - - // Test 1: Authentication requirement - call without auth token - const noAuthOptions: TestOptions = { - ...options, - auth: undefined, // Explicitly remove auth - }; - const noAuthClient = createTestClient(agentUrl, 'mcp', noAuthOptions); - - const { result: noAuthResult, step: noAuthStep } = await runStep( - 'Get products without authentication', - 'get_products', - async () => noAuthClient.executeTask('get_products', { - brief: 'Show me all available products', - brand_manifest: { name: 'Auth Test', url: 'https://test.example.com' }, - }) as Promise - ); - - if (noAuthResult?.success && noAuthResult?.data?.products) { - const products = noAuthResult.data.products as any[]; - noAuthStep.passed = true; - noAuthStep.details = `No auth required: returned ${products.length} product(s)`; - noAuthStep.response_preview = JSON.stringify({ - auth_required: false, - products_returned: products.length, - }, null, 2); - } else if (noAuthResult && !noAuthResult.success) { - noAuthStep.passed = true; - noAuthStep.details = 'Authentication required for get_products'; - noAuthStep.response_preview = JSON.stringify({ - auth_required: true, - error: noAuthResult.error, - }, null, 2); - } else { - noAuthStep.passed = false; - noAuthStep.error = 'Unclear authentication behavior'; - } - steps.push(noAuthStep); - - // Detect if auth is required - if so, subsequent tests need auth to work - const authRequired = noAuthResult && !noAuthResult.success && - noAuthResult.error?.toLowerCase().includes('auth'); - - // Test 2: Brand manifest requirement - call without brand_manifest - const { result: noBrandResult, step: noBrandStep } = await runStep( - 'Get products without brand_manifest', - 'get_products', - async () => authClient.executeTask('get_products', { - brief: 'Show me all available products', - // Intentionally no brand_manifest - }) as Promise - ); - - if (noBrandResult?.success && noBrandResult?.data?.products) { - const products = noBrandResult.data.products as any[]; - noBrandStep.passed = true; - noBrandStep.details = `Brand manifest not required: returned ${products.length} product(s)`; - noBrandStep.response_preview = JSON.stringify({ - brand_manifest_required: false, - products_returned: products.length, - }, null, 2); - } else if (noBrandResult && !noBrandResult.success) { - // Check if this is an auth error (same as the no-auth test) or a brand_manifest error - const isAuthError = noBrandResult.error?.toLowerCase().includes('auth'); - if (isAuthError && authRequired) { - noBrandStep.passed = true; - noBrandStep.details = 'Inconclusive: auth failed (cannot test brand_manifest requirement independently)'; - noBrandStep.response_preview = JSON.stringify({ - brand_manifest_required: 'unknown', - reason: 'Auth token not accepted - cannot isolate brand_manifest requirement', - error: noBrandResult.error, - }, null, 2); - } else { - noBrandStep.passed = true; - noBrandStep.details = 'Brand manifest required for get_products'; - noBrandStep.response_preview = JSON.stringify({ - brand_manifest_required: true, - error: noBrandResult.error, - }, null, 2); - } - } else { - noBrandStep.passed = false; - noBrandStep.error = 'Unclear brand_manifest requirement'; - } - steps.push(noBrandStep); - - // Test 3: Brief relevance - compare generic vs specific briefs - // Skip these tests if auth is required but failing - let genericProductCount = 0; - let specificProductCount = 0; - let briefTestsSkipped = false; - - if (authRequired && noBrandResult && !noBrandResult.success && - noBrandResult.error?.toLowerCase().includes('auth')) { - // Auth is not working - skip brief relevance tests - briefTestsSkipped = true; - const skipStep: TestStepResult = { - step: 'Brief relevance tests', - passed: true, - duration_ms: 0, - details: 'Skipped: auth token not accepted by agent - cannot test brief filtering', - response_preview: JSON.stringify({ - skipped: true, - reason: 'Auth required but token not accepted for get_products', - }, null, 2), - }; - steps.push(skipStep); - } else { - const { result: genericResult, step: genericStep } = await runStep( - 'Get products with generic brief', - 'get_products', - async () => authClient.executeTask('get_products', { - brief: 'Show me all available advertising products', - brand_manifest: { name: 'Brief Test', url: 'https://test.example.com' }, - }) as Promise - ); - - if (genericResult?.success && genericResult?.data?.products) { - const products = genericResult.data.products as any[]; - genericProductCount = products.length; - genericStep.passed = true; - genericStep.details = `Generic brief returned ${products.length} product(s)`; - } else { - genericStep.passed = false; - genericStep.error = genericResult?.error || 'Failed to get products with generic brief'; - } - steps.push(genericStep); - - // Now try a specific brief and compare - const { result: specificResult, step: specificStep } = await runStep( - 'Get products with specific brief', - 'get_products', - async () => authClient.executeTask('get_products', { - brief: 'I need video advertising products for automotive brands targeting luxury car buyers aged 35-55', - brand_manifest: { - name: 'Luxury Auto Brand', - url: 'https://test.example.com', - industry: 'automotive', - target_audience: 'luxury car buyers aged 35-55', - }, - }) as Promise - ); - - if (specificResult?.success && specificResult?.data?.products) { - const products = specificResult.data.products as any[]; - specificProductCount = products.length; - specificStep.passed = true; - specificStep.details = `Specific brief returned ${products.length} product(s)`; - - // Analyze if results are filtered - const channels = new Set(); - for (const product of products) { - if (product.channels) { - for (const ch of product.channels) { - channels.add(ch); - } - } - } - specificStep.response_preview = JSON.stringify({ - products_returned: products.length, - channels: Array.from(channels), - }, null, 2); - } else { - specificStep.passed = false; - specificStep.error = specificResult?.error || 'Failed to get products with specific brief'; - } - steps.push(specificStep); - } - - // Test 4: Analyze filtering behavior based on comparison (skip if auth failed) - if (!briefTestsSkipped) { - const filteringAnalysisStep: TestStepResult = { - step: 'Analyze brief relevance filtering', - passed: true, - duration_ms: 0, - }; - - if (genericProductCount > 0 && specificProductCount > 0) { - if (specificProductCount < genericProductCount) { - filteringAnalysisStep.details = `Agent filters by brief: generic=${genericProductCount}, specific=${specificProductCount} (${Math.round((1 - specificProductCount / genericProductCount) * 100)}% reduction)`; - filteringAnalysisStep.response_preview = JSON.stringify({ - filtering_behavior: 'filtered', - generic_count: genericProductCount, - specific_count: specificProductCount, - reduction_percent: Math.round((1 - specificProductCount / genericProductCount) * 100), - }, null, 2); - } else if (specificProductCount === genericProductCount) { - filteringAnalysisStep.details = `Agent returns same products regardless of brief (${genericProductCount} products)`; - filteringAnalysisStep.response_preview = JSON.stringify({ - filtering_behavior: 'unfiltered', - generic_count: genericProductCount, - specific_count: specificProductCount, - note: 'Same products returned for different briefs', - }, null, 2); - } else { - filteringAnalysisStep.details = `Specific brief returned more products (${specificProductCount} > ${genericProductCount})`; - filteringAnalysisStep.response_preview = JSON.stringify({ - filtering_behavior: 'expanded', - generic_count: genericProductCount, - specific_count: specificProductCount, - note: 'More products for detailed brief - may include related products', - }, null, 2); - } - } else if (genericProductCount === 0 && specificProductCount === 0) { - filteringAnalysisStep.details = 'No products returned for either brief'; - filteringAnalysisStep.passed = false; - filteringAnalysisStep.error = 'Cannot analyze filtering - no products available'; - } else { - filteringAnalysisStep.details = `Partial results: generic=${genericProductCount}, specific=${specificProductCount}`; - } - steps.push(filteringAnalysisStep); - } - - // Test 5: Channel filtering - test if agent filters by requested channel - // Skip if auth is not working - if (!briefTestsSkipped) { - const { result: channelResult, step: channelStep } = await runStep( - 'Get products with channel-specific brief', - 'get_products', - async () => authClient.executeTask('get_products', { - brief: 'I only want display advertising products, no video or audio', - brand_manifest: { name: 'Channel Test', url: 'https://test.example.com' }, - channels: ['display'], // Explicit channel filter if supported - }) as Promise - ); - - if (channelResult?.success && channelResult?.data?.products) { - const products = channelResult.data.products as any[]; - const channels = new Set(); - for (const product of products) { - if (product.channels) { - for (const ch of product.channels) { - channels.add(ch); - } - } - } - - const hasOnlyDisplay = channels.size === 1 && channels.has('display'); - const hasDisplay = channels.has('display'); - - if (hasOnlyDisplay) { - channelStep.details = `Agent correctly filtered to display-only: ${products.length} product(s)`; - } else if (hasDisplay && channels.size > 1) { - channelStep.details = `Agent included display + other channels: ${Array.from(channels).join(', ')}`; - } else { - channelStep.details = `Agent returned channels: ${Array.from(channels).join(', ')} (${products.length} products)`; - } - channelStep.response_preview = JSON.stringify({ - products_returned: products.length, - channels_in_response: Array.from(channels), - display_only: hasOnlyDisplay, - }, null, 2); - } else if (channelResult && !channelResult.success) { - channelStep.passed = true; - channelStep.details = 'Channel filter parameter not supported'; - channelStep.response_preview = JSON.stringify({ error: channelResult.error }, null, 2); - } else { - channelStep.passed = false; - channelStep.error = 'Unclear channel filtering behavior'; - } - steps.push(channelStep); - } - - return { steps, profile }; -} - -/** - * Test: Temporal Validation - * Tests date/time ordering, format validation, and deadline logic - */ -async function testTemporalValidation(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed || !profile.tools.includes('create_media_buy')) { - steps.push({ - step: 'Temporal validation', - passed: false, - duration_ms: 0, - error: 'Agent does not support create_media_buy', - }); - return { steps, profile }; - } - - // Test 1: End time before start time (MUST fail) - const now = Date.now(); - const { result: endBeforeStartResult, step: endBeforeStartStep } = await runStep( - 'End time before start time', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `temporal-test-${Date.now()}`, - brand_manifest: { name: 'Temporal Test', url: 'https://test.example.com' }, - start_time: new Date(now + 604800000).toISOString(), // 7 days from now - end_time: new Date(now + 86400000).toISOString(), // 1 day from now (BEFORE start!) - packages: [{ - buyer_ref: 'pkg-temporal', - product_id: 'test-product', - budget: 1000, - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (endBeforeStartResult && !endBeforeStartResult.success && endBeforeStartResult.error) { - endBeforeStartStep.passed = true; - endBeforeStartStep.details = 'Agent correctly rejected end_time before start_time'; - } else if (endBeforeStartResult?.success) { - endBeforeStartStep.passed = false; - endBeforeStartStep.error = 'CRITICAL: Agent accepted end_time before start_time'; - } else { - endBeforeStartStep.passed = false; - endBeforeStartStep.error = 'Unclear response for invalid temporal ordering'; - } - steps.push(endBeforeStartStep); - - // Test 2: Start time in the past - const { result: pastStartResult, step: pastStartStep } = await runStep( - 'Start time in the past', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `past-start-${Date.now()}`, - brand_manifest: { name: 'Past Start Test', url: 'https://test.example.com' }, - start_time: new Date(now - 604800000).toISOString(), // 7 days ago (IN THE PAST!) - end_time: new Date(now + 604800000).toISOString(), // 7 days from now - packages: [{ - buyer_ref: 'pkg-past', - product_id: 'test-product', - budget: 1000, - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (pastStartResult && !pastStartResult.success && pastStartResult.error) { - pastStartStep.passed = true; - pastStartStep.details = 'Agent rejected start_time in the past'; - } else if (pastStartResult?.success) { - // Some agents may allow past start times for immediate activation - pastStartStep.passed = true; - pastStartStep.details = 'Agent accepts past start_time (immediate activation mode)'; - } else { - pastStartStep.passed = false; - pastStartStep.error = 'Unclear response for past start_time'; - } - steps.push(pastStartStep); - - // Test 3: Invalid ISO 8601 date format - const { result: invalidDateResult, step: invalidDateStep } = await runStep( - 'Invalid date format', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `invalid-date-${Date.now()}`, - brand_manifest: { name: 'Invalid Date Test', url: 'https://test.example.com' }, - start_time: '01/15/2025', // Invalid - should be ISO 8601 - end_time: new Date(now + 604800000).toISOString(), - packages: [{ - buyer_ref: 'pkg-invalid-date', - product_id: 'test-product', - budget: 1000, - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (invalidDateResult && !invalidDateResult.success && invalidDateResult.error) { - invalidDateStep.passed = true; - invalidDateStep.details = 'Agent rejected invalid date format'; - } else if (invalidDateResult?.success) { - invalidDateStep.passed = false; - invalidDateStep.error = 'Agent accepted non-ISO 8601 date format'; - } else { - invalidDateStep.passed = false; - invalidDateStep.error = 'Unclear response for invalid date format'; - } - steps.push(invalidDateStep); - - // Test 4: Very long campaign duration (edge case) - const { result: longCampaignResult, step: longCampaignStep } = await runStep( - 'Very long campaign duration (365 days)', - 'create_media_buy', - async () => client.executeTask('create_media_buy', { - buyer_ref: `long-campaign-${Date.now()}`, - brand_manifest: { name: 'Long Campaign Test', url: 'https://test.example.com' }, - start_time: new Date(now + 86400000).toISOString(), - end_time: new Date(now + 365 * 86400000).toISOString(), // 365 days! - packages: [{ - buyer_ref: 'pkg-long', - product_id: 'test-product', - budget: 100000, - pricing_option_id: 'test-pricing', - }], - }) as Promise - ); - - if (longCampaignResult?.success) { - longCampaignStep.passed = true; - longCampaignStep.details = 'Agent accepts 365-day campaign'; - } else if (longCampaignResult && !longCampaignResult.success && longCampaignResult.error) { - longCampaignStep.passed = true; - longCampaignStep.details = 'Agent has maximum campaign duration limit'; - } else { - longCampaignStep.passed = false; - longCampaignStep.error = 'Unclear response for long campaign'; - } - steps.push(longCampaignStep); - - return { steps, profile }; -} - -/** - * Test: Response Consistency - * Tests for schema errors, pagination bugs, data mismatches between fields - */ -async function testResponseConsistency(agentUrl: string, options: TestOptions): Promise<{ steps: TestStepResult[]; profile?: AgentProfile }> { - const steps: TestStepResult[] = []; - const client = createTestClient(agentUrl, 'mcp', options); - - const { profile, step: profileStep } = await discoverAgentProfile(client); - steps.push(profileStep); - - if (!profileStep.passed) { - return { steps, profile }; - } - - // Test 1: list_creatives pagination consistency - if (profile.tools.includes('list_creatives')) { - const { result, step } = await runStep( - 'list_creatives pagination consistency', - 'list_creatives', - async () => client.executeTask('list_creatives', {}) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const creatives = data.creatives || []; - const querySummary = data.query_summary; - const pagination = data.pagination; - - const issues: string[] = []; - - // Check: total_matching vs returned vs creatives.length - if (querySummary) { - const totalMatching = querySummary.total_matching; - const returned = querySummary.returned; - - if (totalMatching !== undefined && returned !== undefined) { - if (totalMatching > 0 && returned === 0 && creatives.length === 0) { - issues.push(`total_matching=${totalMatching} but returned=0 and creatives array empty`); - } - if (returned !== creatives.length) { - issues.push(`returned=${returned} doesn't match creatives.length=${creatives.length}`); - } - } - } - - // Check: pagination consistency - if (pagination) { - if (pagination.has_more && creatives.length === 0) { - issues.push(`has_more=true but no creatives returned`); - } - if (pagination.total_pages !== undefined && pagination.current_page !== undefined) { - if (pagination.current_page > pagination.total_pages) { - issues.push(`current_page=${pagination.current_page} > total_pages=${pagination.total_pages}`); - } - } - } - - if (issues.length > 0) { - step.passed = false; - step.error = `Pagination inconsistencies: ${issues.join('; ')}`; - step.response_preview = JSON.stringify({ - issues, - query_summary: querySummary, - pagination, - creatives_count: creatives.length, - }, null, 2); - } else { - step.details = `Pagination consistent: ${creatives.length} creative(s)`; - step.response_preview = JSON.stringify({ - creatives_count: creatives.length, - total_matching: querySummary?.total_matching, - returned: querySummary?.returned, - }, null, 2); - } - } else if (result && !result.success) { - // Schema validation error - report it - step.passed = false; - step.error = result.error || 'list_creatives failed'; - } - steps.push(step); - } - - // Test 2: get_products response consistency - if (profile.tools.includes('get_products')) { - const { result, step } = await runStep( - 'get_products response consistency', - 'get_products', - async () => client.executeTask('get_products', { - brief: 'Show all available products', - brand_manifest: { name: 'Consistency Test', url: 'https://test.example.com' }, - }) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const products = data.products || []; - const issues: string[] = []; - - for (let i = 0; i < products.length; i++) { - const product = products[i]; - - // Check: product_id is present and non-empty - if (!product.product_id) { - issues.push(`Product[${i}] missing product_id`); - } - - // Check: pricing_options consistency - if (product.pricing_options) { - for (let j = 0; j < product.pricing_options.length; j++) { - const po = product.pricing_options[j]; - if (!po.pricing_option_id) { - issues.push(`Product[${i}].pricing_options[${j}] missing pricing_option_id`); - } - // Auction pricing should have floor_price or price_guidance - if (po.is_fixed === false && !po.floor_price && !po.price_guidance) { - issues.push(`Product[${i}].pricing_options[${j}] is auction but missing floor_price/price_guidance`); - } - } - } - - // Check: format_ids are structured objects (not plain strings) - if (product.format_ids) { - for (let j = 0; j < product.format_ids.length; j++) { - const fid = product.format_ids[j]; - if (typeof fid === 'string') { - issues.push(`Product[${i}].format_ids[${j}] is string, should be {agent_url, id} object`); - } else if (!fid.id) { - issues.push(`Product[${i}].format_ids[${j}] missing id field`); - } - } - } - } - - if (issues.length > 0) { - step.passed = false; - step.error = `Product data inconsistencies (${issues.length} issue(s))`; - step.response_preview = JSON.stringify({ - issues: issues.slice(0, 10), // Limit to first 10 - total_issues: issues.length, - products_count: products.length, - }, null, 2); - } else { - step.details = `Products consistent: ${products.length} product(s)`; - step.response_preview = JSON.stringify({ - products_count: products.length, - all_have_product_id: true, - all_pricing_options_valid: true, - }, null, 2); - } - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'get_products failed'; - } - steps.push(step); - } - - // Test 3: list_creative_formats response consistency - if (profile.tools.includes('list_creative_formats')) { - const { result, step } = await runStep( - 'list_creative_formats response consistency', - 'list_creative_formats', - async () => client.executeTask('list_creative_formats', {}) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const formatIds = data.format_ids || []; - const formats = data.formats || []; - const issues: string[] = []; - - // Check format_ids structure - for (let i = 0; i < formatIds.length; i++) { - const fid = formatIds[i]; - if (typeof fid === 'string') { - issues.push(`format_ids[${i}] is string "${fid}", should be {agent_url, id} object`); - } else if (fid && !fid.id) { - issues.push(`format_ids[${i}] missing id field`); - } - } - - // Check formats structure if present - for (let i = 0; i < formats.length; i++) { - const fmt = formats[i]; - if (!fmt.format_id?.id) { - issues.push(`formats[${i}] missing format_id.id`); - } - } - - if (issues.length > 0) { - step.passed = false; - step.error = `Format data inconsistencies (${issues.length} issue(s))`; - step.response_preview = JSON.stringify({ - issues: issues.slice(0, 10), - total_issues: issues.length, - format_ids_count: formatIds.length, - formats_count: formats.length, - }, null, 2); - } else { - step.details = `Formats consistent: ${formatIds.length} format_ids, ${formats.length} formats`; - } - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'list_creative_formats failed'; - } - steps.push(step); - } - - // Test 4: list_authorized_properties response consistency - if (profile.tools.includes('list_authorized_properties')) { - const { result, step } = await runStep( - 'list_authorized_properties response consistency', - 'list_authorized_properties', - async () => client.executeTask('list_authorized_properties', {}) as Promise - ); - - if (result?.success && result?.data) { - const data = result.data as any; - const properties = data.authorized_properties || data.properties || []; - const publisherDomains = data.publisher_domains || []; - const issues: string[] = []; - - // Check publisher_domains are strings - for (let i = 0; i < publisherDomains.length; i++) { - if (typeof publisherDomains[i] !== 'string') { - issues.push(`publisher_domains[${i}] is not a string`); - } - } - - // Check properties have required fields - for (let i = 0; i < properties.length; i++) { - const prop = properties[i]; - if (!prop.name && !prop.property_id && !prop.domain) { - issues.push(`properties[${i}] missing identifying field (name, property_id, or domain)`); - } - } - - if (issues.length > 0) { - step.passed = false; - step.error = `Property data inconsistencies (${issues.length} issue(s))`; - step.response_preview = JSON.stringify({ - issues: issues.slice(0, 10), - total_issues: issues.length, - properties_count: properties.length, - publisher_domains_count: publisherDomains.length, - }, null, 2); - } else { - step.details = `Properties consistent: ${properties.length} properties, ${publisherDomains.length} domains`; - } - } else if (result && !result.success) { - step.passed = false; - step.error = result.error || 'list_authorized_properties failed'; - } - steps.push(step); - } - - return { steps, profile }; -} - -/** - * Main entry point: Run a test scenario against an agent - */ -export async function testAgent( - agentUrl: string, - scenario: TestScenario, - options: TestOptions = {} -): Promise { - const startTime = Date.now(); - let steps: TestStepResult[] = []; - let profile: AgentProfile | undefined; - - // Default dry_run to true for safety - const effectiveOptions = { - ...options, - dry_run: options.dry_run !== false, - test_session_id: options.test_session_id || `addie-test-${Date.now()}`, - }; - - logger.info({ agentUrl, scenario, options: effectiveOptions }, 'Starting agent test'); - - try { - let result: { steps: TestStepResult[]; profile?: AgentProfile }; - - switch (scenario) { - case 'health_check': - steps = await testHealthCheck(agentUrl, effectiveOptions); - break; - case 'discovery': - result = await testDiscovery(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'create_media_buy': - result = await testCreateMediaBuy(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'full_sales_flow': - result = await testFullSalesFlow(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'creative_sync': - result = await testCreativeSync(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'creative_inline': - result = await testCreativeInline(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'creative_reference': - // TODO: Implement reference creative testing - steps = [{ - step: 'Test reference creatives', - passed: false, - duration_ms: 0, - error: 'creative_reference scenario not yet implemented', - }]; - break; - case 'pricing_models': - result = await testPricingModels(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'creative_flow': - result = await testCreativeFlow(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'signals_flow': - result = await testSignalsFlow(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'error_handling': - result = await testErrorHandling(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'validation': - result = await testValidation(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'pricing_edge_cases': - result = await testPricingEdgeCases(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'temporal_validation': - result = await testTemporalValidation(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'behavior_analysis': - result = await testBehaviorAnalysis(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - case 'response_consistency': - result = await testResponseConsistency(agentUrl, effectiveOptions); - steps = result.steps; - profile = result.profile; - break; - default: - steps = [{ - step: 'Unknown scenario', - passed: false, - duration_ms: 0, - error: `Unknown test scenario: ${scenario}`, - }]; - } - } catch (error) { - logger.error({ error, agentUrl, scenario }, 'Agent test failed with exception'); - steps.push({ - step: 'Test execution', - passed: false, - duration_ms: Date.now() - startTime, - error: error instanceof Error ? error.message : String(error), - }); - } - - const totalDuration = Date.now() - startTime; - const passedCount = steps.filter(s => s.passed).length; - const failedCount = steps.filter(s => !s.passed).length; - const overallPassed = failedCount === 0 && passedCount > 0; - - // Generate summary - let summary: string; - if (overallPassed) { - summary = `All ${passedCount} test step(s) passed in ${totalDuration}ms`; - } else if (passedCount === 0) { - summary = `All ${failedCount} test step(s) failed`; - } else { - summary = `${passedCount} passed, ${failedCount} failed out of ${steps.length} step(s)`; - } - - const testResult: TestResult = { - agent_url: agentUrl, - scenario, - overall_passed: overallPassed, - steps, - summary, - total_duration_ms: totalDuration, - tested_at: new Date().toISOString(), - agent_profile: profile, - dry_run: effectiveOptions.dry_run, - }; - - logger.info({ agentUrl, scenario, overallPassed, passedCount, failedCount, totalDuration }, 'Agent test completed'); - - return testResult; -} - -/** - * Format test results for display in Slack/chat - */ -export function formatTestResults(result: TestResult): string { - const statusEmoji = result.overall_passed ? '✅' : '❌'; - let output = `## ${statusEmoji} Agent Test Results\n\n`; - output += `**Agent:** ${result.agent_url}\n`; - output += `**Scenario:** ${result.scenario}\n`; - output += `**Duration:** ${result.total_duration_ms}ms\n`; - output += `**Mode:** ${result.dry_run ? '🧪 Dry Run' : '🔴 Live'}\n`; - output += `**Result:** ${result.summary}\n\n`; - - // Show agent profile if discovered - if (result.agent_profile) { - output += `### Agent Capabilities\n`; - output += `- **Name:** ${result.agent_profile.name}\n`; - output += `- **Tools:** ${result.agent_profile.tools.length}\n`; - if (result.agent_profile.channels?.length) { - output += `- **Channels:** ${result.agent_profile.channels.join(', ')}\n`; - } - if (result.agent_profile.pricing_models?.length) { - output += `- **Pricing Models:** ${result.agent_profile.pricing_models.join(', ')}\n`; - } - output += '\n'; - } - - output += `### Test Steps\n\n`; - - for (const step of result.steps) { - const stepEmoji = step.passed ? '✅' : '❌'; - output += `${stepEmoji} **${step.step}**`; - if (step.task) { - output += ` (\`${step.task}\`)`; - } - output += ` - ${step.duration_ms}ms\n`; - - if (step.details) { - output += ` ${step.details}\n`; - } - - if (step.error) { - output += ` ⚠️ Error: ${step.error}\n`; - } - - if (step.response_preview && !step.error) { - output += ` \`\`\`json\n ${step.response_preview.split('\n').join('\n ')}\n \`\`\`\n`; - } - - output += '\n'; - } - - if (!result.overall_passed) { - output += `---\n\n`; - output += `💡 **Need help?** Ask me about specific errors or check the [AdCP documentation](https://adcontextprotocol.org/docs).\n`; - } - - return output; -} diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index f69190f52c..6cdb14b1b7 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -15,12 +15,72 @@ import { logger } from '../../logger.js'; import type { AddieTool } from '../types.js'; import { AdAgentsManager } from '../../adagents-manager.js'; import type { MemberContext } from '../member-context.js'; -import { testAgent, formatTestResults, type TestScenario, type TestOptions } from './agent-tester.js'; +import { + runAgentTests, + formatTestResults, + setAgentTesterLogger, + createTestClient, + type TestScenario, + type TestOptions, +} from '@adcp/client/testing'; import { AgentContextDatabase } from '../../db/agent-context-db.js'; const adagentsManager = new AdAgentsManager(); const agentContextDb = new AgentContextDatabase(); +/** + * Known open-source agents and their GitHub repositories. + * Used to offer GitHub issue links when tests fail on these agents. + * Keys must be lowercase (hostnames are case-insensitive). + */ +const KNOWN_OPEN_SOURCE_AGENTS: Record = { + 'test-agent.adcontextprotocol.org': { + org: 'adcontextprotocol', + repo: 'salesagent', + name: 'AdCP Reference Sales Agent', + }, + 'wonderstruck.sales-agent.scope3.com': { + org: 'adcontextprotocol', + repo: 'salesagent', + name: 'Wonderstruck (Scope3 Sales Agent)', + }, + 'creative.adcontextprotocol.org': { + org: 'adcontextprotocol', + repo: 'creative-agent', + name: 'AdCP Reference Creative Agent', + }, +}; + +/** + * Extract hostname from an agent URL for matching against known agents + */ +function getAgentHostname(agentUrl: string): string | null { + try { + const url = new URL(agentUrl); + return url.hostname; + } catch { + return null; + } +} + +/** + * Check if an agent URL is a known open-source agent + */ +function getOpenSourceAgentInfo(agentUrl: string): { org: string; repo: string; name: string } | null { + const hostname = getAgentHostname(agentUrl); + if (!hostname) return null; + // Normalize to lowercase for case-insensitive matching + return KNOWN_OPEN_SOURCE_AGENTS[hostname.toLowerCase()] || null; +} + +// Configure the agent tester to use our pino logger +setAgentTesterLogger({ + info: (ctx, msg) => logger.info(ctx, msg), + error: (ctx, msg) => logger.error(ctx, msg), + warn: (ctx, msg) => logger.warn(ctx, msg), + debug: (ctx, msg) => logger.debug(ctx, msg), +}); + /** * Tool definitions for member-related operations */ @@ -354,6 +414,38 @@ export const MEMBER_TOOLS: AddieTool[] = [ required: ['agent_url'], }, }, + { + name: 'call_adcp_tool', + description: + 'Call any AdCP task on an agent and return the raw response. Use this for custom testing, exploring agent capabilities, or when you need to call a specific task with specific parameters. This is lower-level than test_adcp_agent - use it when you need precise control over the request/response.', + usage_hints: 'use for "call get_products on my agent", "try create_media_buy with these parameters", "what does list_creative_formats return", exploring agent responses, debugging specific calls', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to call (e.g., "https://sales.example.com/mcp")', + }, + task: { + type: 'string', + description: 'The AdCP task to call (e.g., "get_products", "create_media_buy", "list_creative_formats", "sync_creatives")', + }, + params: { + type: 'object', + description: 'Parameters to pass to the task. Structure depends on the task being called.', + }, + auth_token: { + type: 'string', + description: 'Bearer token for agents that require authentication. Will auto-lookup saved token if not provided.', + }, + dry_run: { + type: 'boolean', + description: 'Whether to include X-Dry-Run header (default: true for safety)', + }, + }, + required: ['agent_url', 'task'], + }, + }, // ============================================ // AGENT CONTEXT MANAGEMENT @@ -1167,7 +1259,7 @@ export function createMemberToolHandlers( if (authToken) options.auth = { type: 'bearer', token: authToken }; try { - const result = await testAgent(agentUrl, scenario, options); + const result = await runAgentTests(agentUrl, scenario, options); // If user is authenticated and agent test succeeded, update the saved context if (organizationId) { @@ -1214,6 +1306,20 @@ export function createMemberToolHandlers( if (usingSavedToken) { output = `_Using saved credentials for this agent._\n\n` + output; } + + // If tests failed on a known open-source agent, offer to help file a GitHub issue + const failedSteps = result.steps.filter((s) => !s.passed); + if (failedSteps.length > 0) { + const openSourceInfo = getOpenSourceAgentInfo(agentUrl); + if (openSourceInfo) { + output += `\n---\n\n`; + output += `💡 **This is an open-source agent** (${openSourceInfo.name})\n\n`; + output += `Since ${failedSteps.length} test step(s) failed, would you like me to help you report this issue?\n`; + output += `I can draft a GitHub issue for the \`${openSourceInfo.org}/${openSourceInfo.repo}\` repository with all the relevant details.\n\n`; + output += `Just say "yes, file an issue" or "help me report this bug" and I'll create a pre-filled GitHub link for you.`; + } + } + return output; } catch (error) { logger.error({ error, agentUrl, scenario }, 'Addie: test_adcp_agent failed'); @@ -1221,6 +1327,79 @@ export function createMemberToolHandlers( } }); + handlers.set('call_adcp_tool', async (input) => { + const agentUrl = input.agent_url as string; + const task = input.task as string; + const params = (input.params as Record) || {}; + const dryRun = input.dry_run as boolean | undefined; + let authToken = input.auth_token as string | undefined; + + // Auto-lookup saved token if user didn't provide one and has org context + let usingSavedToken = false; + const organizationId = memberContext?.organization?.workos_organization_id; + if (!authToken && organizationId) { + try { + const savedToken = await agentContextDb.getAuthTokenByOrgAndUrl(organizationId, agentUrl); + if (savedToken) { + authToken = savedToken; + usingSavedToken = true; + logger.info({ agentUrl, task }, 'Using saved auth token for call_adcp_tool'); + } + } catch (error) { + logger.debug({ error, agentUrl }, 'Could not lookup saved token'); + } + } + + try { + // Create a test client for the agent + const testOptions: TestOptions = { + test_session_id: `addie-call-${Date.now()}`, + dry_run: dryRun, // undefined defaults to true in createTestClient + }; + if (authToken) { + testOptions.auth = { type: 'bearer', token: authToken }; + } + + const client = createTestClient(agentUrl, 'mcp', testOptions); + const startTime = Date.now(); + + // Execute the task + const result = await client.executeTask(task, params); + const duration = Date.now() - startTime; + + // Format response + let output = `## AdCP Task Result\n\n`; + output += `**Agent:** ${agentUrl}\n`; + output += `**Task:** \`${task}\`\n`; + output += `**Duration:** ${duration}ms\n`; + output += `**Mode:** ${dryRun !== false ? '🧪 Dry Run' : '🔴 Live'}\n`; + if (usingSavedToken) { + output += `**Auth:** Using saved credentials\n`; + } + output += `\n`; + + if (result.success) { + output += `### ✅ Success\n\n`; + output += '```json\n'; + output += JSON.stringify(result.data, null, 2); + output += '\n```\n'; + } else { + output += `### ❌ Error\n\n`; + output += `**Error:** ${result.error || 'Unknown error'}\n`; + if (result.data) { + output += '\n**Response data:**\n```json\n'; + output += JSON.stringify(result.data, null, 2); + output += '\n```\n'; + } + } + + return output; + } catch (error) { + logger.error({ error, agentUrl, task }, 'Addie: call_adcp_tool failed'); + return `Failed to call ${task} on ${agentUrl}: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + // ============================================ // GITHUB ISSUE DRAFTING // ============================================ diff --git a/server/tsconfig.json b/server/tsconfig.json index 37c5ef22c0..9e6050d72a 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "ES2022", "lib": ["ES2022"], - "moduleResolution": "node", + "moduleResolution": "bundler", "outDir": "../dist", "rootDir": "./src", "strict": true, From 2e01c3d19bdc8873eee079b4d787e9431b7a0776 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 3 Jan 2026 15:15:40 -0500 Subject: [PATCH 5/8] Add image/iframe support to web chat and update homepage link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update homepage "Test AdCP" button to route to Addie chat (/chat.html) instead of external testing.adcontextprotocol.org - Add image support to chat.html with clickable images that open in new tab - Add iframe support for creative previews with sandbox and styling - Add CSS styles for embedded images and creative preview containers 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- server/public/chat.html | 88 +++++++++++++++++++++++++++++++++++++++- server/public/index.html | 2 +- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/server/public/chat.html b/server/public/chat.html index 8aa2194769..4891b7f703 100644 --- a/server/public/chat.html +++ b/server/public/chat.html @@ -177,6 +177,55 @@ color: white; } + /* Image support */ + .message-content img { + max-width: 100%; + height: auto; + border-radius: 8px; + margin: 8px 0; + display: block; + } + + .message-content img:hover { + cursor: pointer; + } + + /* iframe support for creative previews */ + .message-content .creative-preview-container { + position: relative; + width: 100%; + margin: 12px 0; + border-radius: 8px; + overflow: hidden; + border: 1px solid var(--color-border); + background: var(--color-bg-subtle); + } + + .message-content .creative-preview-container iframe { + width: 100%; + min-height: 200px; + border: none; + display: block; + } + + .message-content .creative-preview-label { + font-size: 11px; + color: var(--color-text-muted); + padding: 4px 8px; + background: var(--color-bg-card); + border-top: 1px solid var(--color-border); + } + + /* Inline HTML creative container */ + .message-content .creative-html-container { + margin: 12px 0; + border-radius: 8px; + overflow: hidden; + border: 1px solid var(--color-border); + background: white; + padding: 16px; + } + .message-content ul, .message-content ol { margin: 8px 0; padding-left: 20px; @@ -707,7 +756,7 @@

Hi! I'm Addie

gfm: true, // GitHub flavored markdown }); - // Use marked's built-in renderer with custom link handling + // Use marked's built-in renderer with custom link and image handling const renderer = new marked.Renderer(); renderer.link = function(href, title, text) { // Handle marked v17+ which passes an object @@ -721,9 +770,46 @@

Hi! I'm Addie

return `${text}`; }; + // Custom image renderer + renderer.image = function(href, title, text) { + // Handle marked v17+ which passes an object + if (typeof href === 'object') { + const img = href; + href = img.href; + title = img.title; + text = img.text; + } + const titleAttr = title ? ` title="${title}"` : ''; + const altAttr = text ? ` alt="${text}"` : ' alt="Image"'; + // Make images clickable to open in new tab + return ``; + }; + return marked.parse(text, { renderer }); } + // Render creative preview (iframe or HTML) + function renderCreativePreview(previewUrl, label) { + if (!previewUrl) return ''; + const safeLabel = label ? label.replace(//g, '>') : 'Creative Preview'; + return ` +
+ +
${safeLabel}
+
+ `; + } + + // Render inline HTML creative + function renderCreativeHtml(html, label) { + if (!html) return ''; + return ` +
+ ${html} +
+ `; + } + // Add message to chat function addMessage(content, role, messageId = null) { // Hide welcome message diff --git a/server/public/index.html b/server/public/index.html index b99cffdf8e..b3e65c9171 100644 --- a/server/public/index.html +++ b/server/public/index.html @@ -256,7 +256,7 @@ -

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

🎯
Built for outcomes
Buy the way you want to grow
🤖
Built for agents
Supports MCP and A2A protocols
🌍
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTry It LiveView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes →

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

✓ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

🛒 Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

🎨 Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

📊 Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

⚡ Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM • 2.3M reach
Platform B: $18 CPM • 4.1M reach
Platform C: $9 CPM • 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source • MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
+

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

🎯
Built for outcomes
Buy the way you want to grow
🤖
Built for agents
Supports MCP and A2A protocols
🌍
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTest with AddieView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes →

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

✓ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

🛒 Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

🎨 Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

📊 Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

⚡ Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM • 2.3M reach
Platform B: $18 CPM • 4.1M reach
Platform C: $9 CPM • 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source • MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
From 5a8feecf1654d89e7075e2ff1f5255be4b5f1396 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 3 Jan 2026 15:30:25 -0500 Subject: [PATCH 6/8] Add "Try AdCP live" prompt and fix migration conflict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add "Try AdCP with a test agent" as suggested prompt in Addie chat (replaces "Become a member" to encourage testing) - Rename 079_agent_contexts.sql to 109_agent_contexts.sql to resolve migration number conflict with main branch 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- server/public/chat.html | 2 +- .../{079_agent_contexts.sql => 109_agent_contexts.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename server/src/db/migrations/{079_agent_contexts.sql => 109_agent_contexts.sql} (100%) diff --git a/server/public/chat.html b/server/public/chat.html index eef83d8f13..0a33094805 100644 --- a/server/public/chat.html +++ b/server/public/chat.html @@ -672,8 +672,8 @@

Hi! I'm Addie

+ -
diff --git a/server/src/db/migrations/079_agent_contexts.sql b/server/src/db/migrations/109_agent_contexts.sql similarity index 100% rename from server/src/db/migrations/079_agent_contexts.sql rename to server/src/db/migrations/109_agent_contexts.sql From cf8e3c9bfb0b7a8b0b946eaa8455ca883b0c965a Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 3 Jan 2026 15:33:30 -0500 Subject: [PATCH 7/8] Add query string prompt parameter to chat.html MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Support ?prompt= query parameter to pre-fill and auto-send a message - Update homepage "Test with Addie" button to include prompt parameter - Waits for Addie to be ready before sending, with 10s timeout - Cleans URL after sending to avoid re-triggering on refresh Now clicking "Test with Addie" on homepage will automatically trigger the test agent flow: /chat.html?prompt=Try%20AdCP%20with%20a%20test%20agent 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- server/public/chat.html | 24 ++++++++++++++++++++++++ server/public/index.html | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/server/public/chat.html b/server/public/chat.html index 0a33094805..ac1a924158 100644 --- a/server/public/chat.html +++ b/server/public/chat.html @@ -1377,9 +1377,33 @@

Hi! I'm Addie

} }); + // Check for prompt in query string (e.g., ?prompt=Try%20AdCP) + function checkQueryPrompt() { + const params = new URLSearchParams(window.location.search); + const prompt = params.get('prompt'); + if (prompt && prompt.trim()) { + // Wait for Addie to be ready before sending + const waitForReady = setInterval(() => { + if (isReady) { + clearInterval(waitForReady); + chatInput.value = prompt.trim(); + autoResize(); + updateSendButton(); + sendMessage(); + // Clean URL without reloading + window.history.replaceState({}, '', window.location.pathname); + } + }, 100); + // Timeout after 10 seconds + setTimeout(() => clearInterval(waitForReady), 10000); + } + } + // Initialize checkStatus(); checkImpersonation(); + // Check for prompt in URL after status check + setTimeout(checkQueryPrompt, 500); // Check status periodically setInterval(checkStatus, 30000); diff --git a/server/public/index.html b/server/public/index.html index b3e65c9171..2854aba212 100644 --- a/server/public/index.html +++ b/server/public/index.html @@ -256,7 +256,7 @@ -

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

🎯
Built for outcomes
Buy the way you want to grow
🤖
Built for agents
Supports MCP and A2A protocols
🌍
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTest with AddieView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes →

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

✓ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

🛒 Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

🎨 Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

📊 Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

⚡ Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM • 2.3M reach
Platform B: $18 CPM • 4.1M reach
Platform C: $9 CPM • 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source • MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
+

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

🎯
Built for outcomes
Buy the way you want to grow
🤖
Built for agents
Supports MCP and A2A protocols
🌍
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTest with AddieView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes →

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

✓ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

🛒 Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

🎨 Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

📊 Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

⚡ Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM • 2.3M reach
Platform B: $18 CPM • 4.1M reach
Platform C: $9 CPM • 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source • MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
From 5d438b466b9acf7891c0879a56bcf085f4e796db Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 3 Jan 2026 15:36:14 -0500 Subject: [PATCH 8/8] Fix check-snippets CI workflow to use .cjs extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script file is check-testable-snippets.cjs but the workflow was referencing check-testable-snippets.js, causing CI failures. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/check-testable-snippets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-testable-snippets.yml b/.github/workflows/check-testable-snippets.yml index 7dea6c929c..e3ef8c2785 100644 --- a/.github/workflows/check-testable-snippets.yml +++ b/.github/workflows/check-testable-snippets.yml @@ -26,7 +26,7 @@ jobs: if [ -s changed_files.txt ]; then echo "📋 Checking documentation changes for testable snippets..." - node scripts/check-testable-snippets.js + node scripts/check-testable-snippets.cjs else echo "✓ No documentation files changed" fi