diff --git a/.changeset/instrument-agent-test-runs.md b/.changeset/instrument-agent-test-runs.md new file mode 100644 index 0000000000..be09f3cfd5 --- /dev/null +++ b/.changeset/instrument-agent-test-runs.md @@ -0,0 +1,4 @@ +--- +--- + +Add `agent_test_runs` table and wire-up instrumentation so Addie can surface staleness-aware agent-testing prompts (#2299 Stage 2). Adds `agent_testing` block to `MemberContext` with `last_test_at`, `last_outcome`, and `total_tests_30d` fields hydrated from the new table. Write calls added to `evaluate_agent_quality` and `run_storyboard` handlers. diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index 4719a34aa7..265d61a1d4 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -75,6 +75,7 @@ import { BrandDatabase } from '../../db/brand-db.js'; import { issueDomainChallenge, verifyDomainChallenge } from '../../services/brand-claim.js'; import { getWorkos } from '../../auth/workos-client.js'; import { resolveUserRole } from '../../utils/resolve-user-role.js'; +import { recordAgentTestRun } from '../../db/agent-test-db.js'; const memberDb = new MemberDatabase(); const agentContextDb = new AgentContextDatabase(); @@ -3584,6 +3585,33 @@ export function createMemberToolHandlers( output += `\nInterpret these results conversationally. Highlight what's working well, identify the most impactful gaps, and suggest concrete next steps.`; + const workosUserIdForRecord = memberContext?.workos_user?.workos_user_id; + if (workosUserIdForRecord) { + const evalOutcome = ((): 'pass' | 'fail' | 'partial' | 'error' => { + switch (result.overall_status) { + case 'passing': return 'pass'; + case 'partial': return 'partial'; + case 'failing': return 'fail'; + default: return 'error'; + } + })(); + recordAgentTestRun({ + workos_user_id: workosUserIdForRecord, + workos_organization_id: memberContext?.organization?.workos_organization_id, + agent_hostname: getAgentHostname(resolved.resolvedUrl), + agent_protocol: 'mcp', + test_kind: 'quality_evaluation', + outcome: evalOutcome, + duration_ms: result.total_duration_ms, + }).then(async () => { + const slackId = memberContext?.slack_user?.slack_user_id; + if (slackId) { + const { invalidateMemberContextCache } = await import('../member-context.js'); + invalidateMemberContextCache(slackId); + } + }).catch(err => logger.warn({ err }, 'Could not record agent test run')); + } + return output; } catch (error) { const msg = error instanceof Error ? error.message : 'Unknown error'; @@ -3968,6 +3996,26 @@ export function createMemberToolHandlers( } if (dryRun) output += ` This was a dry run — no production state was modified.`; + const workosUserIdForStoryboard = memberContext?.workos_user?.workos_user_id; + if (workosUserIdForStoryboard) { + recordAgentTestRun({ + workos_user_id: workosUserIdForStoryboard, + workos_organization_id: memberContext?.organization?.workos_organization_id, + agent_hostname: getAgentHostname(resolved.resolvedUrl), + agent_protocol: 'mcp', + test_kind: storyboardId, + outcome: result.overall_passed ? 'pass' : 'fail', + duration_ms: result.total_duration_ms, + storyboard_id: storyboardId, + }).then(async () => { + const slackId = memberContext?.slack_user?.slack_user_id; + if (slackId) { + const { invalidateMemberContextCache } = await import('../member-context.js'); + invalidateMemberContextCache(slackId); + } + }).catch(err => logger.warn({ err }, 'Could not record storyboard run')); + } + return output; } catch (error) { logger.error({ error, agentUrl: resolved.resolvedUrl, storyboardId }, 'Addie: run_storyboard failed'); diff --git a/server/src/addie/member-context.ts b/server/src/addie/member-context.ts index a31bd9877a..a895608379 100644 --- a/server/src/addie/member-context.ts +++ b/server/src/addie/member-context.ts @@ -25,6 +25,7 @@ import { resolveSlackUserDisplayName } from '../slack/client.js'; import { PERSONA_LABELS } from '../config/personas.js'; import { resolveEffectiveMembership } from '../db/org-filters.js'; import { resolveUserRole } from '../utils/resolve-user-role.js'; +import { getAgentTestingContext } from '../db/agent-test-db.js'; /** Stripe-defined subscription statuses (safe to interpolate into prompts). */ const KNOWN_SUBSCRIPTION_STATUSES = new Set([ @@ -117,22 +118,6 @@ async function getPendingContentForUser( * which is why the cert prompt rule uses a 45-day freshness guard against * `started_at` rather than trying to infer engagement from this field. */ -/** - * Fetch the most recent agent_test_history row for the user. Powers - * the builder-persona "test your agent" prompt rule for builders whose - * last test run is stale. - */ -async function fetchAgentTesting( - workosUserId: string, -): Promise { - const latest = await agentContextDb.getLatestTestForUser(workosUserId); - if (!latest) return { last_test_at: null, last_outcome: null }; - return { - last_test_at: new Date(latest.started_at), - last_outcome: latest.overall_passed ? 'passed' : 'failed', - }; -} - /** * Count of perspectives this user has published, with the most recent * publish timestamp. Drives the "share what you're building" prompt. @@ -468,20 +453,6 @@ export interface MemberContext { last_activity_at: Date; }; - /** - * Most recent agent test the user has run against any of their saved - * agents. Powers the builder-persona "test your agent" prompt for - * builders whose last test is stale. - * - * Tests against the public test agent or unsaved URLs are not - * recorded — they don't count here either, which matches the rule's - * audience (builders who have already saved their seller agent). - */ - agent_testing?: { - last_test_at: Date | null; - last_outcome: 'passed' | 'failed' | null; - }; - /** * The user's published-perspectives footprint. Powers the "share * what you're building" prompt for active members who haven't @@ -512,6 +483,13 @@ export interface MemberContext { last_shown_at: Date | null; suppressed_until: Date | null; }>; + + /** Agent testing history — used for staleness-aware suggested prompts (#2299 Stage 2) */ + agent_testing?: { + last_test_at: Date | null; + total_tests_30d: number; + last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null; + }; } /** @@ -741,13 +719,6 @@ export async function getMemberContext(slackUserId: string): Promise = { pass: 'passed', fail: 'failed', partial: 'partial', error: 'error' }; + const outcomeLabel = context.agent_testing.last_outcome + ? outcomeLabels[context.agent_testing.last_outcome] ?? context.agent_testing.last_outcome + : 'unknown'; + const lastRunLabel = context.agent_testing.last_test_at + ? context.agent_testing.last_test_at.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }) + : 'never'; + lines.push(''); + lines.push(`Agent testing: last run ${lastRunLabel}, outcome ${outcomeLabel}, ${context.agent_testing.total_tests_30d} test(s) in last 30 days.`); + } + // Slack linking status (only relevant for Slack-originated conversations) if (!context.slack_linked && context.slack_user) { lines.push(''); diff --git a/server/src/db/agent-test-db.ts b/server/src/db/agent-test-db.ts new file mode 100644 index 0000000000..8dfb9bebbc --- /dev/null +++ b/server/src/db/agent-test-db.ts @@ -0,0 +1,67 @@ +import { query } from './client.js'; + +export interface AgentTestRunInsert { + workos_user_id: string; + workos_organization_id?: string | null; + agent_hostname?: string | null; + agent_protocol?: 'mcp' | 'a2a' | 'rest' | null; + test_kind: string; + outcome: 'pass' | 'fail' | 'partial' | 'error'; + duration_ms?: number; + storyboard_id?: string | null; + metadata?: Record; +} + +export interface AgentTestingContext { + last_test_at: Date | null; + total_tests_30d: number; + last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null; +} + +export async function recordAgentTestRun(run: AgentTestRunInsert): Promise { + await query( + `INSERT INTO agent_test_runs + (workos_user_id, workos_organization_id, agent_hostname, agent_protocol, + test_kind, outcome, duration_ms, storyboard_id, metadata) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + [ + run.workos_user_id, + run.workos_organization_id ?? null, + run.agent_hostname ?? null, + run.agent_protocol ?? null, + run.test_kind, + run.outcome, + run.duration_ms ?? -1, + run.storyboard_id ?? null, + JSON.stringify(run.metadata ?? {}), + ] + ); +} + +export async function getAgentTestingContext(workosUserId: string): Promise { + const result = await query<{ + last_test_at: Date | null; + last_outcome: string | null; + total_tests_30d: string; + }>( + `SELECT + ran_at AS last_test_at, + outcome AS last_outcome, + (SELECT COUNT(*)::int + FROM agent_test_runs + WHERE workos_user_id = $1 + AND ran_at > NOW() - INTERVAL '30 days') AS total_tests_30d + FROM agent_test_runs + WHERE workos_user_id = $1 + ORDER BY ran_at DESC + LIMIT 1`, + [workosUserId] + ); + if (result.rows.length === 0) return null; + const row = result.rows[0]; + return { + last_test_at: row.last_test_at, + last_outcome: (row.last_outcome ?? null) as AgentTestingContext['last_outcome'], + total_tests_30d: parseInt(row.total_tests_30d, 10), + }; +} diff --git a/server/src/db/migrations/444_agent_test_runs.sql b/server/src/db/migrations/444_agent_test_runs.sql new file mode 100644 index 0000000000..9e7c97e429 --- /dev/null +++ b/server/src/db/migrations/444_agent_test_runs.sql @@ -0,0 +1,22 @@ +-- NOTE: migrations 433_auto_provision_verified_domain.sql and +-- 433_catalog_adagents_lookup_index.sql both use number 433. +-- This migration is numbered 434 to avoid a second collision. +-- The pre-existing 433 duplication should be resolved separately. + +CREATE TABLE agent_test_runs ( + id BIGSERIAL PRIMARY KEY, + workos_user_id TEXT NOT NULL, + workos_organization_id TEXT, + agent_hostname TEXT, + agent_protocol TEXT CHECK (agent_protocol IS NULL OR agent_protocol IN ('mcp', 'a2a', 'rest')), + test_kind TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('pass', 'fail', 'partial', 'error')), + duration_ms INTEGER NOT NULL DEFAULT -1, + storyboard_id TEXT, + ran_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE INDEX idx_agent_test_runs_user_ran_at ON agent_test_runs (workos_user_id, ran_at DESC); +CREATE INDEX idx_agent_test_runs_org_ran_at ON agent_test_runs (workos_organization_id, ran_at DESC) + WHERE workos_organization_id IS NOT NULL;