From b7a937faf2ef4550a15e31a21c7c9a9ccff0dbf3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 14:00:58 +0000 Subject: [PATCH 1/3] feat(addie): instrument agent test runs for staleness-aware prompts (#3254) Adds `agent_test_runs` table and hydrates `agent_testing` on `MemberContext` so #2299 Stage 2 can fire a "Time for a fresh agent test" suggested prompt to builder personas who haven't run a test recently. https://claude.ai/code/session_018QWw9FNTxNgKLejx54HYs2 --- .changeset/instrument-agent-test-runs.md | 4 ++ server/src/addie/mcp/member-tools.ts | 43 ++++++++++++ server/src/addie/member-context.ts | 39 +++++++++++ server/src/db/agent-test-db.ts | 67 +++++++++++++++++++ .../src/db/migrations/442_agent_test_runs.sql | 22 ++++++ 5 files changed, 175 insertions(+) create mode 100644 .changeset/instrument-agent-test-runs.md create mode 100644 server/src/db/agent-test-db.ts create mode 100644 server/src/db/migrations/442_agent_test_runs.sql diff --git a/.changeset/instrument-agent-test-runs.md b/.changeset/instrument-agent-test-runs.md new file mode 100644 index 0000000000..be09f3cfd5 --- /dev/null +++ b/.changeset/instrument-agent-test-runs.md @@ -0,0 +1,4 @@ +--- +--- + +Add `agent_test_runs` table and wire-up instrumentation so Addie can surface staleness-aware agent-testing prompts (#2299 Stage 2). Adds `agent_testing` block to `MemberContext` with `last_test_at`, `last_outcome`, and `total_tests_30d` fields hydrated from the new table. Write calls added to `evaluate_agent_quality` and `run_storyboard` handlers. diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index 4719a34aa7..39adec4a1e 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -18,6 +18,7 @@ import { parseOAuthClientCredentialsInput } from '../../routes/helpers/oauth-cli import { PUBLIC_TEST_AGENT, INTERNAL_PATH_AGENT_URL } from '../../config/test-agent.js'; import type { AddieTool } from '../types.js'; import type { MemberContext } from '../member-context.js'; +import { invalidateMemberContextCache } from '../member-context.js'; import { ToolError } from '../tool-error.js'; import { checkToolRateLimit } from './tool-rate-limiter.js'; import { isUuid } from '../../utils/uuid.js'; @@ -75,6 +76,7 @@ import { BrandDatabase } from '../../db/brand-db.js'; import { issueDomainChallenge, verifyDomainChallenge } from '../../services/brand-claim.js'; import { getWorkos } from '../../auth/workos-client.js'; import { resolveUserRole } from '../../utils/resolve-user-role.js'; +import { recordAgentTestRun } from '../../db/agent-test-db.js'; const memberDb = new MemberDatabase(); const agentContextDb = new AgentContextDatabase(); @@ -3584,6 +3586,30 @@ export function createMemberToolHandlers( output += `\nInterpret these results conversationally. Highlight what's working well, identify the most impactful gaps, and suggest concrete next steps.`; + const workosUserIdForRecord = memberContext?.workos_user?.workos_user_id; + if (workosUserIdForRecord) { + const evalOutcome = ((): 'pass' | 'fail' | 'partial' | 'error' => { + switch (result.overall_status) { + case 'passing': return 'pass'; + case 'partial': return 'partial'; + case 'failing': return 'fail'; + default: return 'error'; + } + })(); + recordAgentTestRun({ + workos_user_id: workosUserIdForRecord, + workos_organization_id: memberContext?.organization?.workos_organization_id, + agent_hostname: getAgentHostname(resolved.resolvedUrl), + agent_protocol: 'mcp', + test_kind: 'quality_evaluation', + outcome: evalOutcome, + duration_ms: result.total_duration_ms, + }).then(() => { + const slackId = memberContext?.slack_user?.slack_user_id; + if (slackId) invalidateMemberContextCache(slackId); + }).catch(err => logger.warn({ err }, 'Could not record agent test run')); + } + return output; } catch (error) { const msg = error instanceof Error ? error.message : 'Unknown error'; @@ -3968,6 +3994,23 @@ export function createMemberToolHandlers( } if (dryRun) output += ` This was a dry run — no production state was modified.`; + const workosUserIdForStoryboard = memberContext?.workos_user?.workos_user_id; + if (workosUserIdForStoryboard) { + recordAgentTestRun({ + workos_user_id: workosUserIdForStoryboard, + workos_organization_id: memberContext?.organization?.workos_organization_id, + agent_hostname: getAgentHostname(resolved.resolvedUrl), + agent_protocol: 'mcp', + test_kind: storyboardId, + outcome: result.overall_passed ? 'pass' : 'fail', + duration_ms: result.total_duration_ms, + storyboard_id: storyboardId, + }).then(() => { + const slackId = memberContext?.slack_user?.slack_user_id; + if (slackId) invalidateMemberContextCache(slackId); + }).catch(err => logger.warn({ err }, 'Could not record storyboard run')); + } + return output; } catch (error) { logger.error({ error, agentUrl: resolved.resolvedUrl, storyboardId }, 'Addie: run_storyboard failed'); diff --git a/server/src/addie/member-context.ts b/server/src/addie/member-context.ts index a31bd9877a..ad8540be3e 100644 --- a/server/src/addie/member-context.ts +++ b/server/src/addie/member-context.ts @@ -25,6 +25,7 @@ import { resolveSlackUserDisplayName } from '../slack/client.js'; import { PERSONA_LABELS } from '../config/personas.js'; import { resolveEffectiveMembership } from '../db/org-filters.js'; import { resolveUserRole } from '../utils/resolve-user-role.js'; +import { getAgentTestingContext } from '../db/agent-test-db.js'; /** Stripe-defined subscription statuses (safe to interpolate into prompts). */ const KNOWN_SUBSCRIPTION_STATUSES = new Set([ @@ -512,6 +513,13 @@ export interface MemberContext { last_shown_at: Date | null; suppressed_until: Date | null; }>; + + /** Agent testing history — used for staleness-aware suggested prompts (#2299 Stage 2) */ + agent_testing?: { + last_test_at: Date | null; + total_tests_30d: number; + last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null; + }; } /** @@ -880,6 +888,15 @@ export async function getMemberContext(slackUserId: string): Promise = { pass: 'passed', fail: 'failed', partial: 'partial', error: 'error' }; + const outcomeLabel = context.agent_testing.last_outcome + ? outcomeLabels[context.agent_testing.last_outcome] ?? context.agent_testing.last_outcome + : 'unknown'; + const lastRunLabel = context.agent_testing.last_test_at + ? context.agent_testing.last_test_at.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }) + : 'never'; + lines.push(''); + lines.push(`Agent testing: last run ${lastRunLabel}, outcome ${outcomeLabel}, ${context.agent_testing.total_tests_30d} test(s) in last 30 days.`); + } + // Slack linking status (only relevant for Slack-originated conversations) if (!context.slack_linked && context.slack_user) { lines.push(''); diff --git a/server/src/db/agent-test-db.ts b/server/src/db/agent-test-db.ts new file mode 100644 index 0000000000..8dfb9bebbc --- /dev/null +++ b/server/src/db/agent-test-db.ts @@ -0,0 +1,67 @@ +import { query } from './client.js'; + +export interface AgentTestRunInsert { + workos_user_id: string; + workos_organization_id?: string | null; + agent_hostname?: string | null; + agent_protocol?: 'mcp' | 'a2a' | 'rest' | null; + test_kind: string; + outcome: 'pass' | 'fail' | 'partial' | 'error'; + duration_ms?: number; + storyboard_id?: string | null; + metadata?: Record; +} + +export interface AgentTestingContext { + last_test_at: Date | null; + total_tests_30d: number; + last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null; +} + +export async function recordAgentTestRun(run: AgentTestRunInsert): Promise { + await query( + `INSERT INTO agent_test_runs + (workos_user_id, workos_organization_id, agent_hostname, agent_protocol, + test_kind, outcome, duration_ms, storyboard_id, metadata) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + [ + run.workos_user_id, + run.workos_organization_id ?? null, + run.agent_hostname ?? null, + run.agent_protocol ?? null, + run.test_kind, + run.outcome, + run.duration_ms ?? -1, + run.storyboard_id ?? null, + JSON.stringify(run.metadata ?? {}), + ] + ); +} + +export async function getAgentTestingContext(workosUserId: string): Promise { + const result = await query<{ + last_test_at: Date | null; + last_outcome: string | null; + total_tests_30d: string; + }>( + `SELECT + ran_at AS last_test_at, + outcome AS last_outcome, + (SELECT COUNT(*)::int + FROM agent_test_runs + WHERE workos_user_id = $1 + AND ran_at > NOW() - INTERVAL '30 days') AS total_tests_30d + FROM agent_test_runs + WHERE workos_user_id = $1 + ORDER BY ran_at DESC + LIMIT 1`, + [workosUserId] + ); + if (result.rows.length === 0) return null; + const row = result.rows[0]; + return { + last_test_at: row.last_test_at, + last_outcome: (row.last_outcome ?? null) as AgentTestingContext['last_outcome'], + total_tests_30d: parseInt(row.total_tests_30d, 10), + }; +} diff --git a/server/src/db/migrations/442_agent_test_runs.sql b/server/src/db/migrations/442_agent_test_runs.sql new file mode 100644 index 0000000000..9e7c97e429 --- /dev/null +++ b/server/src/db/migrations/442_agent_test_runs.sql @@ -0,0 +1,22 @@ +-- NOTE: migrations 433_auto_provision_verified_domain.sql and +-- 433_catalog_adagents_lookup_index.sql both use number 433. +-- This migration is numbered 434 to avoid a second collision. +-- The pre-existing 433 duplication should be resolved separately. + +CREATE TABLE agent_test_runs ( + id BIGSERIAL PRIMARY KEY, + workos_user_id TEXT NOT NULL, + workos_organization_id TEXT, + agent_hostname TEXT, + agent_protocol TEXT CHECK (agent_protocol IS NULL OR agent_protocol IN ('mcp', 'a2a', 'rest')), + test_kind TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('pass', 'fail', 'partial', 'error')), + duration_ms INTEGER NOT NULL DEFAULT -1, + storyboard_id TEXT, + ran_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE INDEX idx_agent_test_runs_user_ran_at ON agent_test_runs (workos_user_id, ran_at DESC); +CREATE INDEX idx_agent_test_runs_org_ran_at ON agent_test_runs (workos_organization_id, ran_at DESC) + WHERE workos_organization_id IS NOT NULL; From 8860533a26ad0f784bd817ec74facec5c455fca4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 22:51:49 +0000 Subject: [PATCH 2/3] fix(addie): lazy-import invalidateMemberContextCache to unblock unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The top-level `import { invalidateMemberContextCache }` on member-tools.ts forced member-context.ts (and transitively auth.ts) to load at module init, triggering the eager WorkOS constructor before vi.mock could intercept it. Removed the static import; both call sites in the fire-and-forget .then() callbacks now use dynamic `await import('../member-context.js')` — same pattern as line 2195 and as the fix applied in #2729. https://claude.ai/code/session_018QWw9FNTxNgKLejx54HYs2 --- server/src/addie/mcp/member-tools.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index 39adec4a1e..265d61a1d4 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -18,7 +18,6 @@ import { parseOAuthClientCredentialsInput } from '../../routes/helpers/oauth-cli import { PUBLIC_TEST_AGENT, INTERNAL_PATH_AGENT_URL } from '../../config/test-agent.js'; import type { AddieTool } from '../types.js'; import type { MemberContext } from '../member-context.js'; -import { invalidateMemberContextCache } from '../member-context.js'; import { ToolError } from '../tool-error.js'; import { checkToolRateLimit } from './tool-rate-limiter.js'; import { isUuid } from '../../utils/uuid.js'; @@ -3604,9 +3603,12 @@ export function createMemberToolHandlers( test_kind: 'quality_evaluation', outcome: evalOutcome, duration_ms: result.total_duration_ms, - }).then(() => { + }).then(async () => { const slackId = memberContext?.slack_user?.slack_user_id; - if (slackId) invalidateMemberContextCache(slackId); + if (slackId) { + const { invalidateMemberContextCache } = await import('../member-context.js'); + invalidateMemberContextCache(slackId); + } }).catch(err => logger.warn({ err }, 'Could not record agent test run')); } @@ -4005,9 +4007,12 @@ export function createMemberToolHandlers( outcome: result.overall_passed ? 'pass' : 'fail', duration_ms: result.total_duration_ms, storyboard_id: storyboardId, - }).then(() => { + }).then(async () => { const slackId = memberContext?.slack_user?.slack_user_id; - if (slackId) invalidateMemberContextCache(slackId); + if (slackId) { + const { invalidateMemberContextCache } = await import('../member-context.js'); + invalidateMemberContextCache(slackId); + } }).catch(err => logger.warn({ err }, 'Could not record storyboard run')); } From d1c675c10b8e4e575b00d20a515fde00e6a18567 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 23:06:42 +0000 Subject: [PATCH 3/3] fix(addie): reconcile agent_testing type and renumber migration 444 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two post-rebase conflicts against main: 1. Remove fetchAgentTesting stub (main's narrow type) — keep this PR's richer AgentTestingContext shape (total_tests_30d + 4-state outcome) which the staleness-prompt rule actually depends on. 2. Renumber 442_agent_test_runs.sql → 444_agent_test_runs.sql; main claimed 442 (addie_prompt_telemetry_clicks) and 443 (agent_test_history_user_id_index) while this branch was in review. https://claude.ai/code/session_01W4Djuhqn6SJJk5qBSPSdTt --- server/src/addie/member-context.ts | 43 ------------------- ..._test_runs.sql => 444_agent_test_runs.sql} | 0 2 files changed, 43 deletions(-) rename server/src/db/migrations/{442_agent_test_runs.sql => 444_agent_test_runs.sql} (100%) diff --git a/server/src/addie/member-context.ts b/server/src/addie/member-context.ts index ad8540be3e..a895608379 100644 --- a/server/src/addie/member-context.ts +++ b/server/src/addie/member-context.ts @@ -118,22 +118,6 @@ async function getPendingContentForUser( * which is why the cert prompt rule uses a 45-day freshness guard against * `started_at` rather than trying to infer engagement from this field. */ -/** - * Fetch the most recent agent_test_history row for the user. Powers - * the builder-persona "test your agent" prompt rule for builders whose - * last test run is stale. - */ -async function fetchAgentTesting( - workosUserId: string, -): Promise { - const latest = await agentContextDb.getLatestTestForUser(workosUserId); - if (!latest) return { last_test_at: null, last_outcome: null }; - return { - last_test_at: new Date(latest.started_at), - last_outcome: latest.overall_passed ? 'passed' : 'failed', - }; -} - /** * Count of perspectives this user has published, with the most recent * publish timestamp. Drives the "share what you're building" prompt. @@ -469,20 +453,6 @@ export interface MemberContext { last_activity_at: Date; }; - /** - * Most recent agent test the user has run against any of their saved - * agents. Powers the builder-persona "test your agent" prompt for - * builders whose last test is stale. - * - * Tests against the public test agent or unsaved URLs are not - * recorded — they don't count here either, which matches the rule's - * audience (builders who have already saved their seller agent). - */ - agent_testing?: { - last_test_at: Date | null; - last_outcome: 'passed' | 'failed' | null; - }; - /** * The user's published-perspectives footprint. Powers the "share * what you're building" prompt for active members who haven't @@ -749,13 +719,6 @@ export async function getMemberContext(slackUserId: string): Promise