feat(addie): instrument agent test runs for staleness-aware prompts

.changeset/instrument-agent-test-runs.md

@@ -0,0 +1,4 @@
+---
+---
+
+Add `agent_test_runs` table and wire-up instrumentation so Addie can surface staleness-aware agent-testing prompts (#2299 Stage 2). Adds `agent_testing` block to `MemberContext` with `last_test_at`, `last_outcome`, and `total_tests_30d` fields hydrated from the new table. Write calls added to `evaluate_agent_quality` and `run_storyboard` handlers.

server/src/addie/mcp/member-tools.ts

@@ -75,6 +75,7 @@ import { BrandDatabase } from '../../db/brand-db.js';
 import { issueDomainChallenge, verifyDomainChallenge } from '../../services/brand-claim.js';
 import { getWorkos } from '../../auth/workos-client.js';
 import { resolveUserRole } from '../../utils/resolve-user-role.js';
+import { recordAgentTestRun } from '../../db/agent-test-db.js';
 
 const memberDb = new MemberDatabase();
 const agentContextDb = new AgentContextDatabase();
@@ -3584,6 +3585,33 @@ export function createMemberToolHandlers(
 
       output += `\nInterpret these results conversationally. Highlight what's working well, identify the most impactful gaps, and suggest concrete next steps.`;
 
+      const workosUserIdForRecord = memberContext?.workos_user?.workos_user_id;
+      if (workosUserIdForRecord) {
+        const evalOutcome = ((): 'pass' | 'fail' | 'partial' | 'error' => {
+          switch (result.overall_status) {
+            case 'passing': return 'pass';
+            case 'partial': return 'partial';
+            case 'failing': return 'fail';
+            default: return 'error';
+          }
+        })();
+        recordAgentTestRun({
+          workos_user_id: workosUserIdForRecord,
+          workos_organization_id: memberContext?.organization?.workos_organization_id,
+          agent_hostname: getAgentHostname(resolved.resolvedUrl),
+          agent_protocol: 'mcp',
+          test_kind: 'quality_evaluation',
+          outcome: evalOutcome,
+          duration_ms: result.total_duration_ms,
+        }).then(async () => {
+          const slackId = memberContext?.slack_user?.slack_user_id;
+          if (slackId) {
+            const { invalidateMemberContextCache } = await import('../member-context.js');
+            invalidateMemberContextCache(slackId);
+          }
+        }).catch(err => logger.warn({ err }, 'Could not record agent test run'));
+      }
+
       return output;
     } catch (error) {
       const msg = error instanceof Error ? error.message : 'Unknown error';
@@ -3968,6 +3996,26 @@ export function createMemberToolHandlers(
       }
       if (dryRun) output += ` This was a dry run — no production state was modified.`;
 
+      const workosUserIdForStoryboard = memberContext?.workos_user?.workos_user_id;
+      if (workosUserIdForStoryboard) {
+        recordAgentTestRun({
+          workos_user_id: workosUserIdForStoryboard,
+          workos_organization_id: memberContext?.organization?.workos_organization_id,
+          agent_hostname: getAgentHostname(resolved.resolvedUrl),
+          agent_protocol: 'mcp',
+          test_kind: storyboardId,
+          outcome: result.overall_passed ? 'pass' : 'fail',
+          duration_ms: result.total_duration_ms,
+          storyboard_id: storyboardId,
+        }).then(async () => {
+          const slackId = memberContext?.slack_user?.slack_user_id;
+          if (slackId) {
+            const { invalidateMemberContextCache } = await import('../member-context.js');
+            invalidateMemberContextCache(slackId);
+          }
+        }).catch(err => logger.warn({ err }, 'Could not record storyboard run'));
+      }
+
       return output;
     } catch (error) {
       logger.error({ error, agentUrl: resolved.resolvedUrl, storyboardId }, 'Addie: run_storyboard failed');

server/src/addie/member-context.ts

@@ -25,6 +25,7 @@ import { resolveSlackUserDisplayName } from '../slack/client.js';
 import { PERSONA_LABELS } from '../config/personas.js';
 import { resolveEffectiveMembership } from '../db/org-filters.js';
 import { resolveUserRole } from '../utils/resolve-user-role.js';
+import { getAgentTestingContext } from '../db/agent-test-db.js';
 
 /** Stripe-defined subscription statuses (safe to interpolate into prompts). */
 const KNOWN_SUBSCRIPTION_STATUSES = new Set([
@@ -117,22 +118,6 @@ async function getPendingContentForUser(
  * which is why the cert prompt rule uses a 45-day freshness guard against
  * `started_at` rather than trying to infer engagement from this field.
  */
-/**
- * Fetch the most recent agent_test_history row for the user. Powers
- * the builder-persona "test your agent" prompt rule for builders whose
- * last test run is stale.
- */
-async function fetchAgentTesting(
-  workosUserId: string,
-): Promise<MemberContext['agent_testing']> {
-  const latest = await agentContextDb.getLatestTestForUser(workosUserId);
-  if (!latest) return { last_test_at: null, last_outcome: null };
-  return {
-    last_test_at: new Date(latest.started_at),
-    last_outcome: latest.overall_passed ? 'passed' : 'failed',
-  };
-}
-
 /**
  * Count of perspectives this user has published, with the most recent
  * publish timestamp. Drives the "share what you're building" prompt.
@@ -468,20 +453,6 @@ export interface MemberContext {
     last_activity_at: Date;
   };
 
-  /**
-   * Most recent agent test the user has run against any of their saved
-   * agents. Powers the builder-persona "test your agent" prompt for
-   * builders whose last test is stale.
-   *
-   * Tests against the public test agent or unsaved URLs are not
-   * recorded — they don't count here either, which matches the rule's
-   * audience (builders who have already saved their seller agent).
-   */
-  agent_testing?: {
-    last_test_at: Date | null;
-    last_outcome: 'passed' | 'failed' | null;
-  };
-
   /**
    * The user's published-perspectives footprint. Powers the "share
    * what you're building" prompt for active members who haven't
@@ -512,6 +483,13 @@ export interface MemberContext {
     last_shown_at: Date | null;
     suppressed_until: Date | null;
   }>;
+
+  /** Agent testing history — used for staleness-aware suggested prompts (#2299 Stage 2) */
+  agent_testing?: {
+    last_test_at: Date | null;
+    total_tests_30d: number;
+    last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null;
+  };
 }
 
 /**
@@ -741,13 +719,6 @@ export async function getMemberContext(slackUserId: string): Promise<MemberConte
       logger.warn({ error, workosUserId }, 'Addie: Failed to load certification context');
     }
 
-    // Latest agent test — drives the "test your agent" prompt for builders.
-    try {
-      context.agent_testing = await fetchAgentTesting(workosUserId);
-    } catch (error) {
-      logger.warn({ error, workosUserId }, 'Addie: Failed to load agent testing context');
-    }
-
     // Published-perspectives footprint — drives the "share what you're building" prompt.
     try {
       context.perspectives = await fetchPerspectives(workosUserId);
@@ -880,6 +851,15 @@ export async function getMemberContext(slackUserId: string): Promise<MemberConte
       }
     }
 
+    try {
+      const agentTesting = await getAgentTestingContext(workosUserId);
+      if (agentTesting) {
+        context.agent_testing = agentTesting;
+      }
+    } catch (error) {
+      logger.warn({ error, workosUserId }, 'Addie: Failed to get agent testing context');
+    }
+
     logger.debug(
       {
         slackUserId,
@@ -972,12 +952,6 @@ async function resolveContextFromLocalDb(
     logger.warn({ error, workosUserId }, 'Addie Web: Failed to load certification context');
   }
 
-  try {
-    context.agent_testing = await fetchAgentTesting(workosUserId);
-  } catch (error) {
-    logger.warn({ error, workosUserId }, 'Addie Web: Failed to load agent testing context');
-  }
-
   try {
     context.perspectives = await fetchPerspectives(workosUserId);
   } catch (error) {
@@ -1137,6 +1111,15 @@ async function resolveContextFromLocalDb(
     }
   }
 
+  try {
+    const agentTesting = await getAgentTestingContext(workosUserId);
+    if (agentTesting) {
+      context.agent_testing = agentTesting;
+    }
+  } catch (error) {
+    logger.warn({ error, workosUserId }, 'Addie Web: Failed to get agent testing context');
+  }
+
   logger.debug(
     {
       workosUserId,
@@ -1495,6 +1478,19 @@ export function formatMemberContextForPrompt(context: MemberContext, channel: 'w
     }
   }
 
+  // Agent testing history
+  if (context.agent_testing) {
+    const outcomeLabels: Record<string, string> = { pass: 'passed', fail: 'failed', partial: 'partial', error: 'error' };
+    const outcomeLabel = context.agent_testing.last_outcome
+      ? outcomeLabels[context.agent_testing.last_outcome] ?? context.agent_testing.last_outcome
+      : 'unknown';
+    const lastRunLabel = context.agent_testing.last_test_at
+      ? context.agent_testing.last_test_at.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' })
+      : 'never';
+    lines.push('');
+    lines.push(`Agent testing: last run ${lastRunLabel}, outcome ${outcomeLabel}, ${context.agent_testing.total_tests_30d} test(s) in last 30 days.`);
+  }
+
   // Slack linking status (only relevant for Slack-originated conversations)
   if (!context.slack_linked && context.slack_user) {
     lines.push('');

server/src/db/agent-test-db.ts

@@ -0,0 +1,67 @@
+import { query } from './client.js';
+
+export interface AgentTestRunInsert {
+  workos_user_id: string;
+  workos_organization_id?: string | null;
+  agent_hostname?: string | null;
+  agent_protocol?: 'mcp' | 'a2a' | 'rest' | null;
+  test_kind: string;
+  outcome: 'pass' | 'fail' | 'partial' | 'error';
+  duration_ms?: number;
+  storyboard_id?: string | null;
+  metadata?: Record<string, unknown>;
+}
+
+export interface AgentTestingContext {
+  last_test_at: Date | null;
+  total_tests_30d: number;
+  last_outcome: 'pass' | 'fail' | 'partial' | 'error' | null;
+}
+
+export async function recordAgentTestRun(run: AgentTestRunInsert): Promise<void> {
+  await query(
+    `INSERT INTO agent_test_runs
+       (workos_user_id, workos_organization_id, agent_hostname, agent_protocol,
+        test_kind, outcome, duration_ms, storyboard_id, metadata)
+     VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+    [
+      run.workos_user_id,
+      run.workos_organization_id ?? null,
+      run.agent_hostname ?? null,
+      run.agent_protocol ?? null,
+      run.test_kind,
+      run.outcome,
+      run.duration_ms ?? -1,
+      run.storyboard_id ?? null,
+      JSON.stringify(run.metadata ?? {}),
+    ]
+  );
+}
+
+export async function getAgentTestingContext(workosUserId: string): Promise<AgentTestingContext | null> {
+  const result = await query<{
+    last_test_at: Date | null;
+    last_outcome: string | null;
+    total_tests_30d: string;
+  }>(
+    `SELECT
+       ran_at  AS last_test_at,
+       outcome AS last_outcome,
+       (SELECT COUNT(*)::int
+          FROM agent_test_runs
+         WHERE workos_user_id = $1
+           AND ran_at > NOW() - INTERVAL '30 days') AS total_tests_30d
+       FROM agent_test_runs
+      WHERE workos_user_id = $1
+      ORDER BY ran_at DESC
+      LIMIT 1`,
+    [workosUserId]
+  );
+  if (result.rows.length === 0) return null;
+  const row = result.rows[0];
+  return {
+    last_test_at: row.last_test_at,
+    last_outcome: (row.last_outcome ?? null) as AgentTestingContext['last_outcome'],
+    total_tests_30d: parseInt(row.total_tests_30d, 10),
+  };
+}

server/src/db/migrations/444_agent_test_runs.sql

@@ -0,0 +1,22 @@
+-- NOTE: migrations 433_auto_provision_verified_domain.sql and
+-- 433_catalog_adagents_lookup_index.sql both use number 433.
+-- This migration is numbered 434 to avoid a second collision.
+-- The pre-existing 433 duplication should be resolved separately.
+
+CREATE TABLE agent_test_runs (
+  id              BIGSERIAL    PRIMARY KEY,
+  workos_user_id  TEXT         NOT NULL,
+  workos_organization_id TEXT,
+  agent_hostname  TEXT,
+  agent_protocol  TEXT         CHECK (agent_protocol IS NULL OR agent_protocol IN ('mcp', 'a2a', 'rest')),
+  test_kind       TEXT         NOT NULL,
+  outcome         TEXT         NOT NULL CHECK (outcome IN ('pass', 'fail', 'partial', 'error')),
+  duration_ms     INTEGER      NOT NULL DEFAULT -1,
+  storyboard_id   TEXT,
+  ran_at          TIMESTAMPTZ  NOT NULL DEFAULT NOW(),
+  metadata        JSONB        NOT NULL DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX idx_agent_test_runs_user_ran_at ON agent_test_runs (workos_user_id, ran_at DESC);
+CREATE INDEX idx_agent_test_runs_org_ran_at  ON agent_test_runs (workos_organization_id, ran_at DESC)
+  WHERE workos_organization_id IS NOT NULL;