diff --git a/.changeset/dashboard-surfaces-verdict-source.md b/.changeset/dashboard-surfaces-verdict-source.md new file mode 100644 index 0000000000..e7fa44366d --- /dev/null +++ b/.changeset/dashboard-surfaces-verdict-source.md @@ -0,0 +1,31 @@ +--- +--- + +Dashboard `/dashboard/agents` surfaces the new `verdict_source` field on the +compliance tile and a per-run "Your test / Heartbeat / Manual / Webhook" +badge in the History panel. PR 2 of the #4247 unification stack — +read-side cleanup that lets owners distinguish their own on-demand +runs from scheduled heartbeat verdicts at a glance. + +**Context.** PR #4250 added `verdict_source` to +`/api/registry/agents/:url/compliance` and `triggered_by` to each row +returned by `/api/registry/agents/:url/compliance/history`. Both fields +were unrendered in the dashboard until this PR. + +**What changes.** + +- Compliance tile shows `Last checked: 3m ago (your test)` / + `(heartbeat)` / `(manual)` / `(webhook)` after the timestamp. Empty + string when `verdict_source` is null (never run). +- History panel renders a colored badge per run row: + - `Your test` (info-blue) for `triggered_by = 'owner_test'` + - `Heartbeat` (neutral) for `triggered_by = 'heartbeat'` + - `Manual` / `Webhook` (neutral) for the other enum values + +No backend changes; this is pure UI surfacing of fields the API already +emits. Pre-PR-1 rows (which only have `'heartbeat'` / `'manual'` / +`'webhook'`) render with the neutral badge — no regression. + +**Out of scope** (PR 3 of #4247): dropping `agent_test_history` and +backfilling owner-triggered rows. Tracked separately so the destructive +migration soaks behind the read-only UI change. diff --git a/.changeset/derive-agent-context-last-test-from-canonical.md b/.changeset/derive-agent-context-last-test-from-canonical.md new file mode 100644 index 0000000000..68668a6c43 --- /dev/null +++ b/.changeset/derive-agent-context-last-test-from-canonical.md @@ -0,0 +1,37 @@ +--- +--- + +PR 4 of the #4247 unification stack. Replaces direct reads of +`agent_contexts.last_test_*` with a view that derives them from +`agent_compliance_runs` — the canonical source PR #4250 unified onto. + +**What changes.** + +- New column `agent_compliance_runs.triggered_org_id` (nullable). Populated + by the owner-test write path in `evaluate_agent_quality` using the + caller's `organizationId`. Heartbeat / manual / webhook writes leave it + NULL — they don't have an org dimension. Without this column, two orgs + that own the same agent URL (e.g. staging and prod orgs of one publisher) + would conflate their test history through a join on `agent_url` alone. +- New view `agent_context_with_latest_test`: `agent_contexts.*` joined to + the latest non-dry-run `agent_compliance_runs` row scoped by + `(triggered_org_id, agent_url)` via `LEFT JOIN LATERAL`, plus a COUNT + scalar subquery for `total_tests_run`. Surfaces the derived fields as + `canonical_last_test_*` so the column-rename in the SELECT is explicit. +- `AgentContextDatabase.getByOrganization`, `getById`, `getByOrgAndUrl` + now SELECT from the view and alias `canonical_last_test_*` → + `last_test_*` so callers see no shape change. + +**Backward compat.** The legacy `agent_contexts.last_test_*` columns stay. +Third-party (non-owner) `recordTest()` writes still update them — that's +the session-scoped audit trail PR 3 of #4247 retained for non-owner runs. +The columns become dead-letter once `agent_test_history` is dropped (gated +on the soak windows in #4247) and `recordTest()` retires in the follow-up +"final cleanup" PR. + +**Index.** `idx_agent_compliance_runs_triggered_org_url_at` on +`(triggered_org_id, agent_url, tested_at DESC)` (partial, only where +`triggered_org_id IS NOT NULL`) supports the view's per-org `DISTINCT ON` +lookup as a single index scan. + +**Stacked on** #4264 (PR 3) → #4263 (PR 2) → #4250 (PR 1). diff --git a/.changeset/final-cleanup-drop-test-history-and-last-test-columns.md b/.changeset/final-cleanup-drop-test-history-and-last-test-columns.md new file mode 100644 index 0000000000..3260854926 --- /dev/null +++ b/.changeset/final-cleanup-drop-test-history-and-last-test-columns.md @@ -0,0 +1,59 @@ +--- +--- + +Final cleanup of the #4247 compliance-state unification stack. Drops +`agent_test_history` and the `agent_contexts.last_test_*` columns now +that owner test runs persist canonically via +`agent_compliance_runs.triggered_by='owner_test'` (PR #4250) and +read-side derivation goes through `agent_context_with_latest_test` +(PR #4268). + +**Pre-merge gate (load-bearing — destructive migration):** + +1. PR #4250 ≥ 14 days live in prod with zero canonical-write incidents. +2. PR #4263 ≥ 7 days live in prod with the dashboard rendering identical + verdicts via the view-derived path. +3. PR #4264's migration 472 has run; row-count delta on staging is ±0 + (every owner-triggered `agent_test_history` row backfilled into + `agent_compliance_runs`). +4. Third-party (`user_id IS NULL`) rows from `agent_test_history` + exported to S3 cold storage. Export evidence committed to the ops + runbook before the migration runs. Reversibility path is the export, + not pg_dump. +5. PR #4268's view + reader migration confirmed working in prod. + +**What this PR does.** + +- **Migration 474.** Redefines `agent_context_summary` view without + references to the dropped table/columns; drops + `agent_contexts.last_test_*` columns; drops `agent_test_history` + table; refreshes `agent_context_with_latest_test` so the view's + `ac.*` projection no longer carries the removed columns. +- **`agent-context-db.ts`.** Removes `recordTest`, `getTestHistory`, + `getLatestTestForUser`, the `AgentTestHistory` interface, and the + `RecordTestInput` interface. The `last_test_*` SET branches in + `update()` go away; the method now refetches via `getById()` after + the UPDATE so derived view fields stay populated. +- **`evaluate_agent_quality`.** The third-party `recordTest()` write + path is removed. Non-owner runs are now session-scoped — they return + results in the response and do not persist. +- **`run_storyboard`.** The `recordTest()` write path is removed. + Single-storyboard runs remain session-scoped (they don't write + canonical state because that would over-state the test coverage of + a single storyboard). A future `triggered_by = 'storyboard_test'` + enum value would expand canonical writes here, but that's a separate + design discussion. + +**Behavior change.** + +- Third-party / non-owner `evaluate_agent_quality` runs against + someone else's agent no longer leave any persistent state in the + registry. Matches the "owner-only canonical writes" policy from + #4247. Stranger-runs return results to the caller in the same + shape; they just don't persist. +- `run_storyboard` runs (any caller) no longer leave persistent state + in the registry. The dashboard's "tested at" timestamps for an org + reflect only `evaluate_agent_quality` runs (which exercise the full + comply suite); single-storyboard runs are exploratory tooling. + +**Stacked on** #4268 (PR 4) → #4264 (PR 3) → #4263 (PR 2) → #4250 (PR 1). diff --git a/.changeset/unify-owner-compliance-writes.md b/.changeset/unify-owner-compliance-writes.md new file mode 100644 index 0000000000..1bdb5c7f9e --- /dev/null +++ b/.changeset/unify-owner-compliance-writes.md @@ -0,0 +1,10 @@ +--- +--- + +PR 1 of 4 in the compliance-state unification initiative (issue #4247): owner-triggered +`evaluate_agent_quality` runs now write to canonical compliance tables +(`agent_compliance_status`, `agent_compliance_runs`, `agent_storyboard_status`) with +`triggered_by = 'owner_test'`, closing the 12-hour gap between owner tests and the +public `/api/registry/agents/:url/compliance` endpoint. Non-owner runs continue +writing to `agent_test_history` (deprecated in PR 3). Adds `'owner_test'` to both +`triggered_by` CHECK constraints via migration 471. diff --git a/.changeset/unify-owner-evaluate-agent-stop-legacy-write.md b/.changeset/unify-owner-evaluate-agent-stop-legacy-write.md new file mode 100644 index 0000000000..c917d80bd2 --- /dev/null +++ b/.changeset/unify-owner-evaluate-agent-stop-legacy-write.md @@ -0,0 +1,33 @@ +--- +--- + +PR 3 of the #4247 unification stack. Two coupled changes: + +**Backfill historical owner-triggered tests into the canonical compliance +tables.** Migration `472_backfill_agent_test_history_to_compliance_runs.sql` +copies every `agent_test_history` row with a `user_id` into +`agent_compliance_runs` as `triggered_by = 'owner_test'` (carrying the +source row id in `observations_json.backfill_source_id` so a re-run is a +no-op via `WHERE NOT EXISTS`). Each agent's most-recent backfilled row +also upserts into `agent_compliance_status` so the dashboard's compliance +tile immediately reflects a real verdict for any agent that was tested +through Addie pre-PR-#4250 and never ran the heartbeat. + +**Stop the dual write for owner runs.** `evaluate_agent_quality` no longer +calls `agentContextDb.recordTest()` when the caller owns the agent — that +path was the dual-write bug #4247 is closing. The legacy `recordTest` call +is retained ONLY for third-party runs so a stranger who tests someone +else's agent still has a session-scoped audit trail in their own +`agent_test_history`. Owner-triggered runs persist exclusively to +canonical state going forward. + +**Out of scope** (deferred to a follow-up after the soak gates): + +- Drop `agent_test_history` table — gated on the 14-day soak from #4250 + deploy + 7-day soak from #4263 + S3 cold-storage export of the + remaining (`user_id IS NULL`) third-party rows. Migration 472 documents + this in its trailing comment. +- Collapse `agent_contexts.last_test_*` into a derived view — PR 4 of + the #4247 stack. + +**Stacked on** #4263 (PR 2 of #4247) → #4250 (PR 1 of #4247). diff --git a/dist/schemas/onboarding-openapi.d.ts b/dist/schemas/onboarding-openapi.d.ts new file mode 100644 index 0000000000..ed29eaf6de --- /dev/null +++ b/dist/schemas/onboarding-openapi.d.ts @@ -0,0 +1,27 @@ +/** + * OpenAPI registrations for the onboarding REST surface. + * + * `POST /api/organizations` has existed in production for a long time but + * has only ever been documented as a private endpoint exercised by the AAO + * dashboard's `/onboarding` form. Surfacing it in the public spec is the + * minimum-surface answer to the storefront-bootstrap question: a + * third-party app holding only a user's OAuth token needs *one* documented + * call to materialize the org, then `POST /api/me/agents` to land an agent + * (which auto-creates the member profile on first call). + * + * Two fields the handler accepts but the public schema deliberately omits: + * + * - `membership_tier` — owned exclusively by the Stripe webhook. Accepting + * it from the caller would let any user stamp tier intent on their org + * row, leaking tier-gated UI state until/unless a real subscription + * overwrites the column. + * - `corporate_domain` — server derives the value from the authenticated + * user's email. Accepting it as a field invited 400s when a caller's + * value disagreed with their email and gave nothing back when it agreed. + * + * Kept in its own module so the spec generator's import graph stays free + * of route handlers (each route file's transitive imports pull in WorkOS + * init, which fails at module load without env vars). + */ +export {}; +//# sourceMappingURL=onboarding-openapi.d.ts.map \ No newline at end of file diff --git a/dist/schemas/onboarding-openapi.d.ts.map b/dist/schemas/onboarding-openapi.d.ts.map new file mode 100644 index 0000000000..9e449fbf0f --- /dev/null +++ b/dist/schemas/onboarding-openapi.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"onboarding-openapi.d.ts","sourceRoot":"","sources":["../../server/src/schemas/onboarding-openapi.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG"} \ No newline at end of file diff --git a/dist/schemas/onboarding-openapi.js b/dist/schemas/onboarding-openapi.js new file mode 100644 index 0000000000..d3e3074486 --- /dev/null +++ b/dist/schemas/onboarding-openapi.js @@ -0,0 +1,135 @@ +/** + * OpenAPI registrations for the onboarding REST surface. + * + * `POST /api/organizations` has existed in production for a long time but + * has only ever been documented as a private endpoint exercised by the AAO + * dashboard's `/onboarding` form. Surfacing it in the public spec is the + * minimum-surface answer to the storefront-bootstrap question: a + * third-party app holding only a user's OAuth token needs *one* documented + * call to materialize the org, then `POST /api/me/agents` to land an agent + * (which auto-creates the member profile on first call). + * + * Two fields the handler accepts but the public schema deliberately omits: + * + * - `membership_tier` — owned exclusively by the Stripe webhook. Accepting + * it from the caller would let any user stamp tier intent on their org + * row, leaking tier-gated UI state until/unless a real subscription + * overwrites the column. + * - `corporate_domain` — server derives the value from the authenticated + * user's email. Accepting it as a field invited 400s when a caller's + * value disagreed with their email and gave nothing back when it agreed. + * + * Kept in its own module so the spec generator's import graph stays free + * of route handlers (each route file's transitive imports pull in WorkOS + * init, which fails at module load without env vars). + */ +import { z } from 'zod'; +import { registry, ErrorSchema } from './registry.js'; +const OrganizationCompanyTypeSchema = z + .enum(['adtech', 'agency', 'brand', 'publisher', 'data', 'ai', 'other']) + .openapi('OrganizationCompanyType', { + description: "Coarse classification of the organization's role in the open ad ecosystem. Drives default verification badges and the member profile's display category.", +}); +const OrganizationRevenueTierSchema = z + .enum(['under_1m', '1m_5m', '5m_50m', '50m_250m', '250m_1b', '1b_plus']) + .openapi('OrganizationRevenueTier', { + description: 'Annual revenue band, USD. Drives membership-tier eligibility for company-tier seats.', +}); +const CreateOrganizationInputSchema = z + .object({ + organization_name: z.string().min(1).max(200).openapi({ + description: "Display name for the organization. Used both as the org row name and (when auto-bootstrapping a member profile via the first agent registration) as the profile's `display_name`.", + example: 'Acme Media', + }), + is_personal: z.boolean().optional().openapi({ + description: 'Set to `true` to create a personal workspace instead of a corporate organization. Personal workspaces skip corporate-domain verification, are limited to one per user, and cannot host the `company_*` membership tiers.', + default: false, + }), + company_type: OrganizationCompanyTypeSchema.optional(), + revenue_tier: OrganizationRevenueTierSchema.optional(), + marketing_opt_in: z.boolean().optional().openapi({ + description: 'Whether the caller opted in to AAO marketing communications. Recorded once per user (not overwritten on subsequent calls). Independent of Terms-of-Service consent, which is recorded server-side from the request context.', + default: false, + }), +}) + .openapi('CreateOrganizationInput', { + description: [ + 'Request body for `POST /api/organizations`.', + "Bootstraps a WorkOS organization, mirrors the caller as `owner`, records the caller's ToS / privacy-policy acceptance, and (for non-personal orgs) inserts an email-verified record into `organization_domains` so subsequent registry calls can skip explicit domain-verification.", + "Membership tier and corporate domain are *not* caller-supplied: the tier is set by the Stripe webhook on subscription events, and the corporate domain is derived from the authenticated user's email.", + ].join('\n\n'), +}); +const CreateOrganizationResponseSchema = z + .object({ + success: z.boolean().optional(), + organization: z + .object({ + id: z.string().openapi({ example: 'org_01HXZAB123' }), + name: z.string().openapi({ example: 'Acme Media' }), + }) + .optional(), + id: z.string().optional().openapi({ + description: "Set on the **prospect-adoption** path: when an org with the user's email domain already exists in a `prospect` state (i.e. the registry pre-recorded it from a brand crawl but no human had claimed it yet), this call adopts that org for the caller instead of creating a new one.", + }), + name: z.string().optional(), + adopted: z.boolean().optional().openapi({ + description: '`true` when the response is the prospect-adoption path. When `true`, no new WorkOS organization was created — the caller is now the owner of an existing prospect record.', + }), +}) + .openapi('CreateOrganizationResponse', { + description: 'Response from `POST /api/organizations`. The body shape varies by path: a fresh creation returns `{ success: true, organization: { id, name } }`; a prospect adoption returns `{ id, name, adopted: true }` directly. Both paths are 2xx; downstream callers should treat any `2xx` as "the org now exists and you are an owner of it" and read whichever id is present.', +}); +registry.registerPath({ + method: 'post', + path: '/api/organizations', + operationId: 'createOrganization', + summary: 'Create or adopt my organization', + description: [ + "Bootstrap the caller's organization explicitly. Use this when the caller wants to control the organization name, `company_type`, `revenue_tier`, or `is_personal` flag before any agents are registered.", + "**Most storefront-style integrations don't need this call** — `POST /api/me/agents` will auto-create an org for a fresh OAuth user (corporate or personal workspace based on the email domain) and surface `org_auto_created: true` in the response. Reach for `POST /api/organizations` only when the auto-derived defaults aren't acceptable.", + 'Three outcomes depending on the caller\'s state:', + "- **Fresh create** (most common): a new WorkOS organization is created, the caller is added as `owner`, the corporate domain is recorded as email-verified, and ToS / privacy-policy acceptance is logged from the request context. Returns `{ success: true, organization: { id, name } }`.", + "- **Prospect adoption**: an organization with the caller's email domain already exists as a `prospect` (the registry pre-recorded it from a brand crawl but no human had claimed it yet). The caller is promoted to `owner` of the existing record instead of forking a duplicate. Returns `{ id, name, adopted: true }`.", + '- **Already-active conflict**: the org exists and is already claimed by another paying member or a previously joined user. Returns `409` with the existing org id so the caller can switch to a join-request flow (`POST /api/organizations/:orgId/join-requests`) instead of trying to register a duplicate.', + 'Tier transitions happen via the billing flow only — there is no `membership_tier` field on this endpoint. After org creation, send the user to `POST /api/checkout-session` (or the dashboard `/membership` page) to start a subscription; the Stripe webhook is the sole writer of `organizations.membership_tier`.', + 'Rate-limited per user: `15` failed attempts per hour; successful calls do not count against the limit so a legitimate registration is never penalized by earlier validation errors.', + ].join('\n\n'), + tags: ['Onboarding'], + security: [{ bearerAuth: [] }, { oauth2: [] }], + request: { + body: { content: { 'application/json': { schema: CreateOrganizationInputSchema } } }, + }, + responses: { + 200: { + description: 'Prospect adoption — an existing prospect organization for this domain was claimed by the caller. Body is `{ id, name, adopted: true }`.', + content: { 'application/json': { schema: CreateOrganizationResponseSchema } }, + }, + 201: { + description: 'New organization created. Body is `{ success: true, organization: { id, name } }`. The caller is the `owner`; the corporate domain is recorded as email-verified for downstream registry calls.', + content: { 'application/json': { schema: CreateOrganizationResponseSchema } }, + }, + 400: { + description: [ + 'One of:', + '- `organization_name` missing or invalid', + '- `company_type` / `revenue_tier` value not in the documented enum', + "- caller is on a personal-email domain (gmail.com, yahoo.com, …) and is trying to register a corporate org — register `is_personal: true` instead", + '- per-user organization cap reached (10 orgs per user)', + ].join('\n'), + content: { 'application/json': { schema: ErrorSchema } }, + }, + 401: { + description: 'Authentication required', + content: { 'application/json': { schema: ErrorSchema } }, + }, + 409: { + description: "An active organization already exists for this caller's email domain. The body includes `existing_org_id` and `existing_org_name`; the caller should switch to the join-request flow rather than retrying.", + content: { 'application/json': { schema: ErrorSchema } }, + }, + 429: { + description: 'Rate limit exceeded — 15 failed attempts per hour per user. Successful calls do not count against the limit.', + content: { 'application/json': { schema: ErrorSchema } }, + }, + }, +}); +//# sourceMappingURL=onboarding-openapi.js.map \ No newline at end of file diff --git a/dist/schemas/onboarding-openapi.js.map b/dist/schemas/onboarding-openapi.js.map new file mode 100644 index 0000000000..9fb894c315 --- /dev/null +++ b/dist/schemas/onboarding-openapi.js.map @@ -0,0 +1 @@ +{"version":3,"file":"onboarding-openapi.js","sourceRoot":"","sources":["../../server/src/schemas/onboarding-openapi.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEtD,MAAM,6BAA6B,GAAG,CAAC;KACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;KACvE,OAAO,CAAC,yBAAyB,EAAE;IAClC,WAAW,EACT,0JAA0J;CAC7J,CAAC,CAAC;AAEL,MAAM,6BAA6B,GAAG,CAAC;KACpC,IAAI,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;KACvE,OAAO,CAAC,yBAAyB,EAAE;IAClC,WAAW,EACT,sFAAsF;CACzF,CAAC,CAAC;AAEL,MAAM,6BAA6B,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;QACpD,WAAW,EACT,mLAAmL;QACrL,OAAO,EAAE,YAAY;KACtB,CAAC;IACF,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC;QAC1C,WAAW,EACT,0NAA0N;QAC5N,OAAO,EAAE,KAAK;KACf,CAAC;IACF,YAAY,EAAE,6BAA6B,CAAC,QAAQ,EAAE;IACtD,YAAY,EAAE,6BAA6B,CAAC,QAAQ,EAAE;IACtD,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC;QAC/C,WAAW,EACT,6NAA6N;QAC/N,OAAO,EAAE,KAAK;KACf,CAAC;CACH,CAAC;KACD,OAAO,CAAC,yBAAyB,EAAE;IAClC,WAAW,EAAE;QACX,6CAA6C;QAC7C,qRAAqR;QACrR,wMAAwM;KACzM,CAAC,IAAI,CAAC,MAAM,CAAC;CACf,CAAC,CAAC;AAEL,MAAM,gCAAgC,GAAG,CAAC;KACvC,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/B,YAAY,EAAE,CAAC;SACZ,MAAM,CAAC;QACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;QACrD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACpD,CAAC;SACD,QAAQ,EAAE;IACb,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC;QAChC,WAAW,EACT,sRAAsR;KACzR,CAAC;IACF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC;QACtC,WAAW,EACT,2KAA2K;KAC9K,CAAC;CACH,CAAC;KACD,OAAO,CAAC,4BAA4B,EAAE;IACrC,WAAW,EACT,0WAA0W;CAC7W,CAAC,CAAC;AAEL,QAAQ,CAAC,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM;IACd,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,oBAAoB;IACjC,OAAO,EAAE,iCAAiC;IAC1C,WAAW,EAAE;QACX,0MAA0M;QAC1M,iVAAiV;QACjV,kDAAkD;QAClD,8RAA8R;QAC9R,2TAA2T;QAC3T,+SAA+S;QAC/S,sTAAsT;QACtT,qLAAqL;KACtL,CAAC,IAAI,CAAC,MAAM,CAAC;IACd,IAAI,EAAE,CAAC,YAAY,CAAC;IACpB,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC9C,OAAO,EAAE;QACP,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,6BAA6B,EAAE,EAAE,EAAE;KACrF;IACD,SAAS,EAAE;QACT,GAAG,EAAE;YACH,WAAW,EACT,yIAAyI;YAC3I,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE;SAC9E;QACD,GAAG,EAAE;YACH,WAAW,EACT,iMAAiM;YACnM,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE;SAC9E;QACD,GAAG,EAAE;YACH,WAAW,EAAE;gBACX,SAAS;gBACT,0CAA0C;gBAC1C,oEAAoE;gBACpE,mJAAmJ;gBACnJ,wDAAwD;aACzD,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE;SACzD;QACD,GAAG,EAAE;YACH,WAAW,EAAE,yBAAyB;YACtC,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE;SACzD;QACD,GAAG,EAAE;YACH,WAAW,EACT,4MAA4M;YAC9M,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE;SACzD;QACD,GAAG,EAAE;YACH,WAAW,EACT,8GAA8G;YAChH,OAAO,EAAE,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE;SACzD;KACF;CACF,CAAC,CAAC"} \ No newline at end of file diff --git a/server/public/dashboard-agents.html b/server/public/dashboard-agents.html index ae39c1d0a8..f8b9521219 100644 --- a/server/public/dashboard-agents.html +++ b/server/public/dashboard-agents.html @@ -1501,6 +1501,19 @@

Agents

? timeAgo(new Date(cs.last_checked_at)) : 'never'; + // Surface the verdict source so the operator knows whether the + // current status came from the scheduled heartbeat or their own + // owner-triggered test run. PR #4250 populates cs.verdict_source + // ('heartbeat' | 'owner_test' | 'manual' | 'webhook' | null when + // never run). Displayed inline with "Last checked" so the + // semantic shift on the public compliance contract is visible + // to the operator without having to read the changelog. + const verdictSourceLabel = cs.verdict_source === 'owner_test' ? ' (your test)' + : cs.verdict_source === 'heartbeat' ? ' (heartbeat)' + : cs.verdict_source === 'manual' ? ' (manual)' + : cs.verdict_source === 'webhook' ? ' (webhook)' + : ''; + const isPublic = cs.status !== 'opted_out'; return ` @@ -1527,7 +1540,7 @@

Agents

${visibilitySelectorHtml}
- Last checked: ${escapeHtml(lastChecked)} + Last checked: ${escapeHtml(lastChecked)}${escapeHtml(verdictSourceLabel)}
'; if (runTracks) { html += '
' + runTracks + '
'; diff --git a/server/src/addie/config-version.ts b/server/src/addie/config-version.ts index f184b82fcd..bf00b2c4d6 100644 --- a/server/src/addie/config-version.ts +++ b/server/src/addie/config-version.ts @@ -30,7 +30,7 @@ import { loadRules, loadResponseStyle } from './rules/index.js'; * Format: YYYY.MM.N where N is incremented for multiple changes in a month * Example: 2025.01.1, 2025.01.2, 2025.02.1 */ -export const CODE_VERSION = '2026.04.6'; +export const CODE_VERSION = '2026.05.1'; // Types export interface ConfigVersion { diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index a37b108c1d..515190ee73 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -34,6 +34,7 @@ import { SAMPLE_BRIEFS, classifyCapabilityResolutionError, presentCapabilityResolutionError, + complianceResultToDbInput, type ComplyOptions, type ComplianceTrack, } from '../services/compliance-testing.js'; @@ -3559,28 +3560,70 @@ export function createMemberToolHandlers( ); } - // Record result if the user has an org with this agent saved + // Record result when the user has an org context for this agent. if (organizationId) { - try { - const context = await agentContextDb.getByOrgAndUrl(organizationId, resolved.resolvedUrl); - if (context) { - await agentContextDb.recordTest({ - agent_context_id: context.id, - scenario: 'quality_evaluation', - overall_passed: result.overall_status === 'passing', - steps_passed: result.summary.tracks_passed, - steps_failed: result.summary.tracks_failed, - total_duration_ms: result.total_duration_ms, - summary: result.summary.headline, - dry_run: true, - triggered_by: 'user', - user_id: memberContext?.workos_user?.workos_user_id, - agent_profile_json: result.agent_profile, - }); + // Write to canonical compliance tables when the calling org owns this agent. + // Mirrors resolveAgentOwnerOrg (registry-api.ts:4733) — joins organization_memberships + // to verify the acting user is still an active member of the owning org. + // Non-owner runs skip the canonical write and fall through to the legacy + // agent_test_history path below. + const workosUserId = memberContext?.workos_user?.workos_user_id; + let isAgentOwner = false; + if (workosUserId) { + try { + const ownerCheck = await query( + `SELECT 1 FROM member_profiles mp + JOIN organization_memberships om + ON om.workos_organization_id = mp.workos_organization_id + WHERE mp.workos_organization_id = $1 + AND mp.agents @> $2::jsonb + AND om.workos_user_id = $3 + LIMIT 1`, + [organizationId, JSON.stringify([{ url: resolved.resolvedUrl }]), workosUserId], + ); + isAgentOwner = ownerCheck.rows.length > 0; + } catch (ownerCheckError) { + logger.warn({ ownerCheckError }, 'evaluate_agent_quality: owner check failed, skipping canonical write'); + } + } + + if (isAgentOwner) { + try { + const metadata = await complianceDb.getRegistryMetadata(resolved.resolvedUrl); + // Skip canonical write if the owner has opted out of compliance monitoring. + if (!metadata?.compliance_opt_out) { + const dbInput = { + ...complianceResultToDbInput( + result, + resolved.resolvedUrl, + metadata?.lifecycle_stage ?? 'production', + 'owner_test', + ), + // Owner test runs are not dry runs — they update the live public record. + // (complianceResultToDbInput hard-codes dry_run: true; override here.) + dry_run: false, + // Org scope for the per-org `agent_context_with_latest_test` view. + // Without this, two orgs that own the same agent URL (staging vs + // prod orgs of one publisher) would conflate their test history. + // See migration 473. + triggered_org_id: organizationId, + }; + await complianceDb.recordComplianceRun(dbInput); + // notifyComplianceChange intentionally omitted: owner test runs are + // exploratory; compliance-change notifications fire on heartbeat + // transitions only to prevent iteration-loop spam. + } + } catch (error) { + logger.warn({ error, agentUrl: resolved.resolvedUrl }, 'Could not write owner test result to canonical compliance state'); } - } catch (error) { - logger.debug({ error }, 'Could not record quality evaluation result'); } + + // Non-owner runs are now session-scoped — they return results to + // the caller in the response and do not persist anywhere. The legacy + // recordTest path that wrote to agent_test_history was dropped in + // migration 474 (#4247 final cleanup). Strangers testing someone + // else's agent no longer leave persistent state in the registry, + // matching the "owner-only canonical writes" policy from #4247. } // Build structured output for Addie to interpret @@ -4054,33 +4097,18 @@ export function createMemberToolHandlers( return `Agent at ${resolved.resolvedUrl} requires authentication. Use \`save_agent\` to store credentials first, then try again.`; } - // Record the run in agent_test_history when we have a saved - // agent_context for this org+url. Mirrors evaluate_agent_quality's - // pattern; powers the "agent not tested in 14d" prompt rule. - // Storyboard runs don't carry a structured agent_profile (only - // evaluate_agent_quality probes get_adcp_capabilities), so we - // omit agent_profile_json — readers tolerate null. - if (organizationId) { - try { - const context = await agentContextDb.getByOrgAndUrl(organizationId, resolved.resolvedUrl); - if (context) { - await agentContextDb.recordTest({ - agent_context_id: context.id, - scenario: `storyboard:${sb.id}`, - overall_passed: result.overall_passed, - steps_passed: result.passed_count, - steps_failed: result.failed_count, - total_duration_ms: result.total_duration_ms, - summary: result.storyboard_title, - dry_run: dryRun, - triggered_by: 'user', - user_id: memberContext?.workos_user?.workos_user_id, - }); - } - } catch (error) { - logger.debug({ error }, 'Could not record storyboard run'); - } - } + // run_storyboard runs a single storyboard (vs evaluate_agent_quality's + // full comply suite). Single-storyboard runs do not currently write + // canonical state — that would require synthesizing a comply-shaped + // result with one track, which over-states the test coverage. The + // legacy recordTest call that wrote to agent_test_history was dropped + // in migration 474 (#4247 final cleanup); single-storyboard runs are + // exploratory and remain session-scoped. + // + // If a future track wants to surface storyboard runs in + // agent_compliance_runs as a distinct triggered_by value (e.g. + // 'storyboard_test'), open a follow-up — that's a schema change with + // its own design discussion. let output = ''; if (resolved.source === 'saved') output += '_Using saved credentials._\n\n'; diff --git a/server/src/db/agent-context-db.ts b/server/src/db/agent-context-db.ts index 164fcda049..edd1fe118e 100644 --- a/server/src/db/agent-context-db.ts +++ b/server/src/db/agent-context-db.ts @@ -35,7 +35,11 @@ export interface AgentContext { // Discovery cache tools_discovered: string[] | null; last_discovered_at: Date | null; - // Test history + // Test history — derived from agent_compliance_runs via the + // `agent_context_with_latest_test` view (migration 473). The underlying + // columns on agent_contexts were dropped in migration 474; readers + // continue to see the same field names because the view aliases the + // canonical_* columns back to the original names. last_test_scenario: string | null; last_test_passed: boolean | null; last_test_summary: string | null; @@ -81,24 +85,10 @@ export interface OAuthClientCredentials { auth_method?: 'basic' | 'body'; } -export interface AgentTestHistory { - id: string; - agent_context_id: string; - scenario: string; - overall_passed: boolean; - steps_passed: number; - steps_failed: number; - total_duration_ms: number | null; - summary: string | null; - dry_run: boolean; - brief: string | null; - triggered_by: string | null; - user_id: string | null; - steps_json: any; - agent_profile_json: any; - started_at: Date; - completed_at: Date | null; -} +// AgentTestHistory interface and RecordTestInput removed in migration 474. +// agent_test_history table dropped; owner test results live in +// agent_compliance_runs (canonical) via evaluate_agent_quality's +// owner-test write path. export interface CreateAgentContextInput { organization_id: string; @@ -114,25 +104,9 @@ export interface UpdateAgentContextInput { agent_type?: AgentType; protocol?: Protocol; tools_discovered?: string[]; - last_test_scenario?: string; - last_test_passed?: boolean; - last_test_summary?: string; -} - -export interface RecordTestInput { - agent_context_id: string; - scenario: string; - overall_passed: boolean; - steps_passed: number; - steps_failed: number; - total_duration_ms?: number; - summary?: string; - dry_run?: boolean; - brief?: string; - triggered_by?: string; - user_id?: string; - steps_json?: any; - agent_profile_json?: any; + // last_test_* fields removed — derived from agent_compliance_runs via + // the agent_context_with_latest_test view (migrations 473 + 474). + // Owner test runs write to canonical state via recordComplianceRun. } function getTokenHint(token: string, authType: AuthType = 'bearer'): string { @@ -182,15 +156,20 @@ export class AgentContextDatabase { AND oauth_cc_client_secret_encrypted IS NOT NULL) as has_oauth_client_credentials, tools_discovered, last_discovered_at, - last_test_scenario, - last_test_passed, - last_test_summary, - last_tested_at, - total_tests_run, + -- Derived from agent_compliance_runs via the view (canonical source). + -- The legacy agent_contexts.last_test_* columns stay for backward + -- compat with non-owner third-party writes through recordTest, but + -- reads come from the view so the unification is consistent. See + -- migration 473. + canonical_last_test_scenario AS last_test_scenario, + canonical_last_test_passed AS last_test_passed, + canonical_last_test_summary AS last_test_summary, + canonical_last_tested_at AS last_tested_at, + canonical_total_tests_run AS total_tests_run, created_at, updated_at, created_by - FROM agent_contexts + FROM agent_context_with_latest_test WHERE organization_id = $1 ORDER BY updated_at DESC`, [organizationId] @@ -222,15 +201,20 @@ export class AgentContextDatabase { AND oauth_cc_client_secret_encrypted IS NOT NULL) as has_oauth_client_credentials, tools_discovered, last_discovered_at, - last_test_scenario, - last_test_passed, - last_test_summary, - last_tested_at, - total_tests_run, + -- Derived from agent_compliance_runs via the view (canonical source). + -- The legacy agent_contexts.last_test_* columns stay for backward + -- compat with non-owner third-party writes through recordTest, but + -- reads come from the view so the unification is consistent. See + -- migration 473. + canonical_last_test_scenario AS last_test_scenario, + canonical_last_test_passed AS last_test_passed, + canonical_last_test_summary AS last_test_summary, + canonical_last_tested_at AS last_tested_at, + canonical_total_tests_run AS total_tests_run, created_at, updated_at, created_by - FROM agent_contexts + FROM agent_context_with_latest_test WHERE id = $1`, [id] ); @@ -261,15 +245,20 @@ export class AgentContextDatabase { AND oauth_cc_client_secret_encrypted IS NOT NULL) as has_oauth_client_credentials, tools_discovered, last_discovered_at, - last_test_scenario, - last_test_passed, - last_test_summary, - last_tested_at, - total_tests_run, + -- Derived from agent_compliance_runs via the view (canonical source). + -- The legacy agent_contexts.last_test_* columns stay for backward + -- compat with non-owner third-party writes through recordTest, but + -- reads come from the view so the unification is consistent. See + -- migration 473. + canonical_last_test_scenario AS last_test_scenario, + canonical_last_test_passed AS last_test_passed, + canonical_last_test_summary AS last_test_summary, + canonical_last_tested_at AS last_tested_at, + canonical_total_tests_run AS total_tests_run, created_at, updated_at, created_by - FROM agent_contexts + FROM agent_context_with_latest_test WHERE organization_id = $1 AND agent_url = $2`, [organizationId, agentUrl] ); @@ -305,11 +294,16 @@ export class AgentContextDatabase { FALSE as has_oauth_client, tools_discovered, last_discovered_at, - last_test_scenario, - last_test_passed, - last_test_summary, - last_tested_at, - total_tests_run, + -- Derived from agent_compliance_runs via the view (canonical source). + -- The legacy agent_contexts.last_test_* columns stay for backward + -- compat with non-owner third-party writes through recordTest, but + -- reads come from the view so the unification is consistent. See + -- migration 473. + canonical_last_test_scenario AS last_test_scenario, + canonical_last_test_passed AS last_test_passed, + canonical_last_test_summary AS last_test_summary, + canonical_last_tested_at AS last_tested_at, + canonical_total_tests_run AS total_tests_run, created_at, updated_at, created_by`, @@ -350,18 +344,10 @@ export class AgentContextDatabase { updates.push(`last_discovered_at = NOW()`); values.push(input.tools_discovered); } - if (input.last_test_scenario !== undefined) { - updates.push(`last_test_scenario = $${paramIndex++}`); - values.push(input.last_test_scenario); - } - if (input.last_test_passed !== undefined) { - updates.push(`last_test_passed = $${paramIndex++}`); - values.push(input.last_test_passed); - } - if (input.last_test_summary !== undefined) { - updates.push(`last_test_summary = $${paramIndex++}`); - values.push(input.last_test_summary); - } + // last_test_* SET branches removed — those columns no longer exist on + // agent_contexts (migration 474). Owner test runs persist via + // recordComplianceRun; the values flow back through + // agent_context_with_latest_test on read. if (updates.length === 0) { return this.getById(id); @@ -370,40 +356,17 @@ export class AgentContextDatabase { updates.push('updated_at = NOW()'); values.push(id); - const result = await query( + // UPDATE the table, then read back through the view so derived + // last_test_* fields are populated. RETURNING off the bare table no + // longer carries those fields, and refetching via getById is the + // cleanest way to keep the AgentContext shape stable for callers. + await query( `UPDATE agent_contexts SET ${updates.join(', ')} - WHERE id = $${paramIndex} - RETURNING - id, - organization_id, - agent_url, - agent_name, - agent_type, - protocol, - auth_token_encrypted IS NOT NULL as has_auth_token, - auth_token_hint, - auth_type, - oauth_access_token_encrypted IS NOT NULL as has_oauth_token, - oauth_refresh_token_encrypted IS NOT NULL as has_oauth_refresh_token, - oauth_token_expires_at, - oauth_client_id IS NOT NULL as has_oauth_client, - (oauth_cc_token_endpoint IS NOT NULL - AND oauth_cc_client_id IS NOT NULL - AND oauth_cc_client_secret_encrypted IS NOT NULL) as has_oauth_client_credentials, - tools_discovered, - last_discovered_at, - last_test_scenario, - last_test_passed, - last_test_summary, - last_tested_at, - total_tests_run, - created_at, - updated_at, - created_by`, + WHERE id = $${paramIndex}`, values ); - return result.rows[0] || null; + return this.getById(id); } /** @@ -886,103 +849,15 @@ export class AgentContextDatabase { return (result.rowCount ?? 0) > 0; } - /** - * Record a test run - */ - async recordTest(input: RecordTestInput): Promise { - // Update the agent context - await query( - `UPDATE agent_contexts - SET - last_test_scenario = $1, - last_test_passed = $2, - last_test_summary = $3, - last_tested_at = NOW(), - total_tests_run = total_tests_run + 1, - updated_at = NOW() - WHERE id = $4`, - [input.scenario, input.overall_passed, input.summary || null, input.agent_context_id] - ); - - // Insert history record - const result = await query( - `INSERT INTO agent_test_history ( - agent_context_id, - scenario, - overall_passed, - steps_passed, - steps_failed, - total_duration_ms, - summary, - dry_run, - brief, - triggered_by, - user_id, - steps_json, - agent_profile_json, - completed_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW()) - RETURNING *`, - [ - input.agent_context_id, - input.scenario, - input.overall_passed, - input.steps_passed, - input.steps_failed, - input.total_duration_ms || null, - input.summary || null, - input.dry_run ?? true, - input.brief || null, - input.triggered_by || null, - input.user_id || null, - input.steps_json ? JSON.stringify(input.steps_json) : null, - input.agent_profile_json ? JSON.stringify(input.agent_profile_json) : null, - ] - ); - - return result.rows[0]; - } - - /** - * Get test history for an agent - */ - async getTestHistory(agentContextId: string, limit: number = 20): Promise { - const result = await query( - `SELECT * - FROM agent_test_history - WHERE agent_context_id = $1 - ORDER BY started_at DESC - LIMIT $2`, - [agentContextId, limit] - ); - return result.rows; - } - - /** - * Most recent agent test the user has run, across all of their saved - * agents. Powers the "agent not tested in X days" suggested-prompts - * rule for builder personas. - * - * Returns null if the user has never tested any registered agent. - * Tests against the public test agent or unsaved URLs are not - * recorded in agent_test_history (they don't have an agent_context), - * so they don't count here either — which is the right semantic for - * the rule's audience (builders with their own seller agent). - */ - async getLatestTestForUser(workosUserId: string): Promise<{ - started_at: Date; - overall_passed: boolean; - } | null> { - const result = await query<{ started_at: Date; overall_passed: boolean }>( - `SELECT started_at, overall_passed - FROM agent_test_history - WHERE user_id = $1 - ORDER BY started_at DESC - LIMIT 1`, - [workosUserId] - ); - return result.rows[0] ?? null; - } + // recordTest, getTestHistory, getLatestTestForUser removed in migration 474. + // + // Owner-triggered test runs persist via complianceDb.recordComplianceRun + // with triggered_by='owner_test' (PR #4250). Read-side query for "latest + // test by this user/org" goes through the agent_context_with_latest_test + // view (PR #4268), which derives last_tested_at from + // agent_compliance_runs scoped by (triggered_org_id, agent_url). + // Third-party (non-owner) runs are session-scoped only — they return + // results in the response and do not persist anywhere. /** * Infer agent type from discovered tools diff --git a/server/src/db/compliance-db.ts b/server/src/db/compliance-db.ts index 236abff905..94c857f9d2 100644 --- a/server/src/db/compliance-db.ts +++ b/server/src/db/compliance-db.ts @@ -11,7 +11,7 @@ const logger = baseLogger.child({ module: 'compliance-db' }); export type LifecycleStage = 'development' | 'testing' | 'production' | 'deprecated'; export type ComplianceStatus = 'passing' | 'degraded' | 'failing' | 'unknown'; export type OverallRunStatus = 'passing' | 'failing' | 'partial'; -export type TriggeredBy = 'heartbeat' | 'manual' | 'webhook'; +export type TriggeredBy = 'heartbeat' | 'manual' | 'webhook' | 'owner_test'; export type TrackStatus = 'pass' | 'fail' | 'partial' | 'skip' | 'silent'; /** @@ -118,6 +118,8 @@ export interface AgentComplianceStatus { previous_status: string | null; status_changed_at: Date | null; updated_at: Date; + /** triggered_by of the most recent non-dry-run in agent_compliance_runs */ + last_triggered_by: TriggeredBy | null; } export type StoryboardStatus = 'passing' | 'failing' | 'partial' | 'untested'; @@ -182,6 +184,14 @@ export interface RecordComplianceRunInput { agent_profile_json?: any; observations_json?: any; triggered_by?: TriggeredBy; + /** + * WorkOS organization id of the org that triggered the run. Populated only + * for triggered_by='owner_test'; heartbeat / manual / webhook leave it NULL. + * Required for the per-org scoping of `agent_context_with_latest_test` so + * two orgs that own the same agent (e.g. staging vs prod orgs of one + * publisher) don't conflate their test history. See migration 473. + */ + triggered_org_id?: string | null; dry_run?: boolean; storyboard_statuses?: StoryboardStatusEntry[]; } @@ -263,8 +273,8 @@ export class ComplianceDatabase { agent_url, lifecycle_stage, overall_status, headline, total_duration_ms, tracks_json, tracks_passed, tracks_failed, tracks_skipped, tracks_partial, agent_profile_json, - observations_json, triggered_by, dry_run - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + observations_json, triggered_by, triggered_org_id, dry_run + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *`, [ input.agent_url, @@ -280,6 +290,7 @@ export class ComplianceDatabase { input.agent_profile_json ? JSON.stringify(input.agent_profile_json) : null, input.observations_json ? JSON.stringify(input.observations_json) : null, input.triggered_by ?? 'heartbeat', + input.triggered_org_id ?? null, input.dry_run ?? true, ], ); @@ -427,9 +438,15 @@ export class ComplianceDatabase { async getComplianceStatus(agentUrl: string): Promise { const result = await query( - `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage + `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage, + r.triggered_by AS last_triggered_by FROM agent_compliance_status s LEFT JOIN agent_registry_metadata m ON m.agent_url = s.agent_url + LEFT JOIN LATERAL ( + SELECT triggered_by FROM agent_compliance_runs + WHERE agent_url = s.agent_url AND dry_run = false + ORDER BY tested_at DESC LIMIT 1 + ) r ON true WHERE s.agent_url = $1`, [agentUrl], ); @@ -455,9 +472,15 @@ export class ComplianceDatabase { if (agentUrls.length === 0) return new Map(); const result = await query( - `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage + `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage, + r.triggered_by AS last_triggered_by FROM agent_compliance_status s LEFT JOIN agent_registry_metadata m ON m.agent_url = s.agent_url + LEFT JOIN LATERAL ( + SELECT triggered_by FROM agent_compliance_runs + WHERE agent_url = s.agent_url AND dry_run = false + ORDER BY tested_at DESC LIMIT 1 + ) r ON true WHERE s.agent_url = ANY($1)`, [agentUrls], ); diff --git a/server/src/db/migrations/471_owner_test_triggered_by.sql b/server/src/db/migrations/471_owner_test_triggered_by.sql new file mode 100644 index 0000000000..0e450bb1ad --- /dev/null +++ b/server/src/db/migrations/471_owner_test_triggered_by.sql @@ -0,0 +1,16 @@ +-- Add 'owner_test' to triggered_by CHECK constraints in compliance tables. +-- Owner-triggered storyboard runs (via evaluate_agent_quality) now write to +-- canonical compliance state, distinguished from heartbeat and dashboard-manual +-- runs by triggered_by = 'owner_test'. See issue #4247. + +ALTER TABLE agent_compliance_runs + DROP CONSTRAINT IF EXISTS valid_triggered_by, + ADD CONSTRAINT valid_triggered_by CHECK ( + triggered_by IN ('heartbeat', 'manual', 'webhook', 'owner_test') + ); + +ALTER TABLE agent_storyboard_status + DROP CONSTRAINT IF EXISTS valid_storyboard_triggered_by, + ADD CONSTRAINT valid_storyboard_triggered_by CHECK ( + triggered_by IS NULL OR triggered_by IN ('heartbeat', 'manual', 'webhook', 'owner_test') + ); diff --git a/server/src/db/migrations/472_backfill_agent_test_history_to_compliance_runs.sql b/server/src/db/migrations/472_backfill_agent_test_history_to_compliance_runs.sql new file mode 100644 index 0000000000..7f53b2f376 --- /dev/null +++ b/server/src/db/migrations/472_backfill_agent_test_history_to_compliance_runs.sql @@ -0,0 +1,149 @@ +-- Migration 472: backfill owner-triggered agent_test_history rows into +-- agent_compliance_runs as triggered_by='owner_test' rows. +-- +-- Part of the #4247 compliance-state unification (PR 3 of 4). PR #4250 +-- (PR 1) made evaluate_agent_quality write canonical for owner runs going +-- forward; this migration backfills the historical rows so the compliance +-- API and dashboard reflect the full test history, not just runs from the +-- PR #4250 deploy onward. +-- +-- Scope: backfill ONLY rows with user_id IS NOT NULL (real owner-triggered +-- tests). Third-party / scheduled / unattributed rows are NOT touched here +-- — the table drop is a separate follow-up that includes an S3 cold-storage +-- export of those rows so audit history isn't silently lost (see #4247 +-- Acceptance Criteria). +-- +-- Mapping: +-- agent_test_history.agent_context_id → agent_contexts.agent_url +-- agent_test_history.overall_passed → overall_status ('passing' | 'failing') +-- agent_test_history.steps_passed → tracks_passed +-- agent_test_history.steps_failed → tracks_failed +-- agent_test_history.total_duration_ms→ total_duration_ms +-- agent_test_history.summary → headline +-- agent_test_history.agent_profile_json → agent_profile_json +-- agent_test_history.started_at → tested_at +-- triggered_by → 'owner_test' (constant) +-- dry_run → false (PR #4250's owner path uses dry_run=false) +-- +-- Idempotency: backfilled rows carry the source agent_test_history.id in +-- observations_json.{backfill_source} so a re-run is a no-op via the +-- WHERE NOT EXISTS guard. + +INSERT INTO agent_compliance_runs ( + agent_url, + lifecycle_stage, + overall_status, + headline, + total_duration_ms, + tracks_json, + tracks_passed, + tracks_failed, + tracks_skipped, + tracks_partial, + agent_profile_json, + observations_json, + triggered_by, + dry_run, + tested_at +) +SELECT + ac.agent_url, + COALESCE(arm.lifecycle_stage, 'production') AS lifecycle_stage, + CASE WHEN ath.overall_passed THEN 'passing' ELSE 'failing' END AS overall_status, + ath.summary AS headline, + ath.total_duration_ms, + '[]'::jsonb AS tracks_json, + COALESCE(ath.steps_passed, 0) AS tracks_passed, + COALESCE(ath.steps_failed, 0) AS tracks_failed, + 0 AS tracks_skipped, + 0 AS tracks_partial, + ath.agent_profile_json, + jsonb_build_object( + 'backfill_source', 'agent_test_history', + 'backfill_source_id', ath.id::text, + 'backfill_migration', '472', + 'original_scenario', ath.scenario + ) AS observations_json, + 'owner_test' AS triggered_by, + FALSE AS dry_run, + ath.started_at AS tested_at +FROM agent_test_history ath +JOIN agent_contexts ac ON ac.id = ath.agent_context_id +LEFT JOIN agent_registry_metadata arm ON arm.agent_url = ac.agent_url +WHERE ath.user_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM agent_compliance_runs acr + WHERE acr.observations_json->>'backfill_source_id' = ath.id::text + ); + +-- Update agent_compliance_status from the latest backfilled row per agent +-- so the dashboard immediately reflects the most recent owner-triggered +-- verdict for any agent that didn't yet have a heartbeat row. Skipped +-- when a status row already exists from a more recent heartbeat — heartbeat +-- always wins on freshness, last-write-wins is the contract pinned in +-- PR #4250's tests. +INSERT INTO agent_compliance_status ( + agent_url, + status, + lifecycle_stage, + last_checked_at, + last_passed_at, + last_failed_at, + tracks_summary_json, + headline, + status_changed_at, + last_triggered_by +) +SELECT DISTINCT ON (acr.agent_url) + acr.agent_url, + CASE WHEN acr.overall_status = 'passing' THEN 'passing' ELSE 'failing' END, + acr.lifecycle_stage, + acr.tested_at, + CASE WHEN acr.overall_status = 'passing' THEN acr.tested_at ELSE NULL END, + CASE WHEN acr.overall_status = 'failing' THEN acr.tested_at ELSE NULL END, + '{}'::jsonb, + acr.headline, + acr.tested_at, + 'owner_test' +FROM agent_compliance_runs acr +WHERE acr.observations_json->>'backfill_migration' = '472' +ORDER BY acr.agent_url, acr.tested_at DESC +ON CONFLICT (agent_url) DO UPDATE SET + status = CASE + WHEN agent_compliance_status.last_checked_at IS NULL + OR agent_compliance_status.last_checked_at < EXCLUDED.last_checked_at + THEN EXCLUDED.status + ELSE agent_compliance_status.status + END, + last_checked_at = GREATEST( + COALESCE(agent_compliance_status.last_checked_at, EXCLUDED.last_checked_at), + EXCLUDED.last_checked_at + ), + last_passed_at = CASE + WHEN EXCLUDED.last_passed_at IS NOT NULL + AND (agent_compliance_status.last_passed_at IS NULL + OR agent_compliance_status.last_passed_at < EXCLUDED.last_passed_at) + THEN EXCLUDED.last_passed_at + ELSE agent_compliance_status.last_passed_at + END, + last_failed_at = CASE + WHEN EXCLUDED.last_failed_at IS NOT NULL + AND (agent_compliance_status.last_failed_at IS NULL + OR agent_compliance_status.last_failed_at < EXCLUDED.last_failed_at) + THEN EXCLUDED.last_failed_at + ELSE agent_compliance_status.last_failed_at + END, + last_triggered_by = CASE + WHEN agent_compliance_status.last_checked_at IS NULL + OR agent_compliance_status.last_checked_at < EXCLUDED.last_checked_at + THEN EXCLUDED.last_triggered_by + ELSE agent_compliance_status.last_triggered_by + END; + +-- NOTE: this migration does NOT drop agent_test_history. The drop is +-- deferred to a follow-up migration that runs after: +-- (a) the 14-day soak window from PR #4250 deploy, +-- (b) the 7-day soak window from PR #4263 deploy, +-- (c) S3 cold-storage export of third-party rows (user_id IS NULL), +-- (d) row-count delta verification on staging. +-- See #4247 acceptance criteria. diff --git a/server/src/db/migrations/473_agent_compliance_runs_triggered_org_id.sql b/server/src/db/migrations/473_agent_compliance_runs_triggered_org_id.sql new file mode 100644 index 0000000000..7c07a2435a --- /dev/null +++ b/server/src/db/migrations/473_agent_compliance_runs_triggered_org_id.sql @@ -0,0 +1,77 @@ +-- Migration 473: add triggered_org_id to agent_compliance_runs and create +-- agent_context_with_latest_test view. +-- +-- Part of the #4247 compliance-state unification (PR 4 of 4). Today, +-- agent_contexts.last_test_* columns carry the most-recent test verdict +-- per (organization_id, agent_url). After PR #4250 owner runs write +-- canonical state via agent_compliance_runs, but that table tracks only +-- agent_url — there's no org dimension, so a derived "latest owner test" +-- can't be accurately scoped per org. Two orgs that own the same agent +-- (rare but possible — staging vs prod org of one publisher, for example) +-- would conflate. +-- +-- Adding triggered_org_id closes the gap. Populated by the owner-test +-- write path in evaluate_agent_quality (this PR). Heartbeat / manual / +-- webhook writes leave it NULL. + +ALTER TABLE agent_compliance_runs + ADD COLUMN IF NOT EXISTS triggered_org_id TEXT; + +-- Index supports the derived `agent_context_with_latest_test` view's +-- per-(org, url) DISTINCT ON lookup. tested_at DESC keeps the latest-row +-- pull as a single index scan. +CREATE INDEX IF NOT EXISTS idx_agent_compliance_runs_triggered_org_url_at + ON agent_compliance_runs (triggered_org_id, agent_url, tested_at DESC) + WHERE triggered_org_id IS NOT NULL; + +-- View: agent_context joined with the latest agent_compliance_runs row +-- scoped to that org via triggered_org_id. Replaces direct reads of +-- agent_contexts.last_test_* columns. +-- +-- The columns on agent_contexts stay for backward compat — recordTest() +-- still writes them for third-party (non-owner) runs, and a follow-up +-- migration drops them once recordTest() retires (gated on the +-- agent_test_history drop, which is itself gated on the soak windows +-- documented in #4247). +-- +-- last_test_passed: derived from overall_status='passing'. +-- last_test_scenario: tracks_json[0].track when present, else 'compliance' +-- (heartbeat/manual writes don't carry the legacy 'quality_evaluation' +-- scenario string — the closest semantic in the canonical schema is the +-- first track of the run). +-- last_test_summary: agent_compliance_runs.headline. +-- last_tested_at: agent_compliance_runs.tested_at. +-- total_tests_run: COUNT(*) of agent_compliance_runs rows scoped to the +-- org+url. Was a per-context counter on the old column; the COUNT-based +-- derivation matches the new canonical semantics. +CREATE OR REPLACE VIEW agent_context_with_latest_test AS +SELECT + ac.*, + latest.tested_at AS canonical_last_tested_at, + latest.overall_status = 'passing' AS canonical_last_test_passed, + COALESCE( + (latest.tracks_json -> 0 ->> 'track'), + 'compliance' + ) AS canonical_last_test_scenario, + latest.headline AS canonical_last_test_summary, + COALESCE(run_counts.total, 0) AS canonical_total_tests_run +FROM agent_contexts ac +LEFT JOIN LATERAL ( + SELECT tested_at, overall_status, tracks_json, headline + FROM agent_compliance_runs acr + WHERE acr.triggered_org_id = ac.organization_id + AND acr.agent_url = ac.agent_url + AND acr.dry_run = FALSE + ORDER BY tested_at DESC + LIMIT 1 +) AS latest ON TRUE +LEFT JOIN LATERAL ( + SELECT COUNT(*)::INT AS total + FROM agent_compliance_runs acr + WHERE acr.triggered_org_id = ac.organization_id + AND acr.agent_url = ac.agent_url + AND acr.dry_run = FALSE +) AS run_counts ON TRUE; + +COMMENT ON VIEW agent_context_with_latest_test IS + 'Derives last_test_* fields from agent_compliance_runs (triggered_org_id-scoped). Replaces direct reads of agent_contexts.last_test_*. The legacy columns stay for backward compat until recordTest() retires (#4247 PR-after-drop).'; diff --git a/server/src/db/migrations/474_drop_agent_test_history_and_last_test_columns.sql b/server/src/db/migrations/474_drop_agent_test_history_and_last_test_columns.sql new file mode 100644 index 0000000000..b6f1d738e7 --- /dev/null +++ b/server/src/db/migrations/474_drop_agent_test_history_and_last_test_columns.sql @@ -0,0 +1,126 @@ +-- Migration 474: drop agent_test_history table + agent_contexts.last_test_* +-- columns. Final cleanup of the #4247 compliance-state unification stack. +-- +-- Pre-merge gate (load-bearing — DO NOT RUN until each is satisfied): +-- +-- 1. PR #4250 has been live in prod for ≥ 14 days with zero canonical-write +-- incidents (no malformed agent_compliance_status row, no flap reports). +-- 2. PR #4263 has been live in prod for ≥ 7 days with the dashboard +-- rendering identical verdicts via the view-derived path. +-- 3. PR #4264's migration 472 has run and the row-count delta on staging +-- is ±0 (every owner-triggered agent_test_history row backfilled into +-- agent_compliance_runs). +-- 4. Third-party (`user_id IS NULL`) rows from agent_test_history have +-- been exported to S3 cold storage. Export evidence committed to the +-- ops runbook before this migration runs. Do NOT silently lose audit +-- history. +-- 5. PR #4268's view + reader migration confirmed working in prod +-- (all callers of last_test_* read from agent_context_with_latest_test +-- via the column-aliased SELECTs). +-- +-- This migration is destructive and irreversible. Reversibility path is +-- the S3 export from gate (4), not pg_dump. + +-- ── Phase 1: drop the dependent view, redefining without legacy columns ── + +DROP VIEW IF EXISTS agent_context_summary; + +CREATE OR REPLACE VIEW agent_context_summary AS +SELECT + ac.id, + ac.organization_id, + ac.agent_url, + ac.agent_name, + ac.agent_type, + ac.protocol, + ac.auth_token_hint, + ac.auth_token_encrypted IS NOT NULL as has_auth_token, + ac.oauth_access_token_encrypted IS NOT NULL as has_oauth_token, + ac.oauth_token_expires_at, + ac.oauth_client_id IS NOT NULL as has_oauth_client, + ac.tools_discovered, + -- last_test_* fields now derived from agent_compliance_runs scoped to + -- (triggered_org_id, agent_url) rather than read off the legacy + -- agent_contexts columns. Mirrors agent_context_with_latest_test + -- (migration 473) so callers can stop using the legacy view if they + -- prefer the explicit canonical-source name. + v.canonical_last_test_scenario AS last_test_scenario, + v.canonical_last_test_passed AS last_test_passed, + v.canonical_last_test_summary AS last_test_summary, + v.canonical_last_tested_at AS last_tested_at, + v.canonical_total_tests_run AS total_tests_run, + ac.created_at, + ac.updated_at, + -- history_count / history_passed_count from agent_test_history removed — + -- the canonical-runs derivation gives total_tests_run, and a history + -- count of "passing runs" is inferable from the per-org rollup of + -- agent_compliance_runs if needed. + v.canonical_total_tests_run AS history_count, + COALESCE(( + SELECT COUNT(*) + FROM agent_compliance_runs acr + WHERE acr.triggered_org_id = ac.organization_id + AND acr.agent_url = ac.agent_url + AND acr.dry_run = FALSE + AND acr.overall_status = 'passing' + ), 0) AS history_passed_count +FROM agent_contexts ac +LEFT JOIN agent_context_with_latest_test v ON v.id = ac.id; + +COMMENT ON VIEW agent_context_summary IS + 'Agent contexts with auth info and last_test_* derived from agent_compliance_runs (canonical). Replaces the migration 195 definition that referenced the dropped agent_test_history.'; + +-- ── Phase 2: drop legacy columns from agent_contexts ── + +-- agent_context_with_latest_test (migration 473) selects ac.* — once these +-- columns drop, ac.* simply omits them. The view's aliased +-- canonical_last_test_* fields stay intact (they come from the LATERAL +-- JOIN against agent_compliance_runs). + +ALTER TABLE agent_contexts DROP COLUMN IF EXISTS last_test_scenario; +ALTER TABLE agent_contexts DROP COLUMN IF EXISTS last_test_passed; +ALTER TABLE agent_contexts DROP COLUMN IF EXISTS last_test_summary; +ALTER TABLE agent_contexts DROP COLUMN IF EXISTS last_tested_at; +ALTER TABLE agent_contexts DROP COLUMN IF EXISTS total_tests_run; + +-- ── Phase 3: drop agent_test_history ── + +-- Indexes drop with the table. +-- ON DELETE CASCADE FK from agent_contexts.id is severed when the table +-- drops; agent_contexts is unaffected. + +DROP TABLE IF EXISTS agent_test_history; + +-- ── Phase 4: refresh agent_context_with_latest_test ── +-- The view (from migration 473) selects ac.* — once last_test_* columns +-- are gone, the view's projection no longer includes them. CREATE OR +-- REPLACE re-binds the view definition cleanly. + +CREATE OR REPLACE VIEW agent_context_with_latest_test AS +SELECT + ac.*, + latest.tested_at AS canonical_last_tested_at, + latest.overall_status = 'passing' AS canonical_last_test_passed, + COALESCE( + (latest.tracks_json -> 0 ->> 'track'), + 'compliance' + ) AS canonical_last_test_scenario, + latest.headline AS canonical_last_test_summary, + COALESCE(run_counts.total, 0) AS canonical_total_tests_run +FROM agent_contexts ac +LEFT JOIN LATERAL ( + SELECT tested_at, overall_status, tracks_json, headline + FROM agent_compliance_runs acr + WHERE acr.triggered_org_id = ac.organization_id + AND acr.agent_url = ac.agent_url + AND acr.dry_run = FALSE + ORDER BY tested_at DESC + LIMIT 1 +) AS latest ON TRUE +LEFT JOIN LATERAL ( + SELECT COUNT(*)::INT AS total + FROM agent_compliance_runs acr + WHERE acr.triggered_org_id = ac.organization_id + AND acr.agent_url = ac.agent_url + AND acr.dry_run = FALSE +) AS run_counts ON TRUE; diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 925aec6cda..936b7b5dc0 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -4253,6 +4253,7 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { membership_tier_label: ownerMembership.membership_tier_label, subscription_status: ownerMembership.subscription_status, is_api_access_tier: ownerMembership.is_api_access_tier, + verdict_source: status.last_triggered_by ?? null, verified: badges.length > 0, verified_badges: badges.map(b => ({ role: b.role, diff --git a/server/src/schemas/registry.ts b/server/src/schemas/registry.ts index 4a59bdd2be..f29a961f11 100644 --- a/server/src/schemas/registry.ts +++ b/server/src/schemas/registry.ts @@ -341,6 +341,8 @@ export const AgentComplianceDetailSchema = z membership_tier_label: z.string().nullable().optional().openapi({ description: "Owner-scoped: human-readable label for membership_tier (e.g. 'Builder'). Null for non-owners." }), subscription_status: z.string().nullable().optional().openapi({ description: "Owner-scoped: the agent owner's subscription status (active, past_due, trialing, etc.). Null for non-owners." }), is_api_access_tier: z.boolean().optional().openapi({ description: "Owner-scoped: true when the owner's tier and subscription status grant badge eligibility. False for non-owners. Single source of truth — UI should not re-derive." }), + verdict_source: z.enum(["heartbeat", "owner_test", "manual", "webhook"]).nullable().optional() + .openapi({ description: "triggered_by value of the most recent non-dry-run compliance check. 'heartbeat' = scheduled run; 'owner_test' = agent owner triggered via evaluate_agent_quality. Null when no run has been recorded yet." }), verified: z.boolean().optional(), verified_badges: z.array(VerificationBadgeSchema).optional(), }) diff --git a/server/tests/unit/compliance-db-last-write-wins.test.ts b/server/tests/unit/compliance-db-last-write-wins.test.ts new file mode 100644 index 0000000000..bec028e4bc --- /dev/null +++ b/server/tests/unit/compliance-db-last-write-wins.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../src/db/client.js', () => ({ + query: vi.fn(), + getClient: vi.fn(), +})); + +vi.mock('../../src/db/encryption.js', () => ({ + decrypt: vi.fn(), + encrypt: vi.fn(), + deriveKey: vi.fn(), +})); + +import { ComplianceDatabase } from '../../src/db/compliance-db.js'; +import { query, getClient } from '../../src/db/client.js'; + +const mockedQuery = vi.mocked(query); +const mockedGetClient = vi.mocked(getClient); + +const EMPTY = { rows: [], rowCount: 0, command: '', oid: 0, fields: [] }; + +function makeTransactionClient(queryResponses: Array<{ rows: any[] }>) { + const calls: string[] = []; + let idx = 0; + const client = { + query: vi.fn(async (sql: string) => { + calls.push(typeof sql === 'string' ? sql.trim().split(/\s+/)[0] : sql); + const resp = queryResponses[idx] ?? EMPTY; + idx++; + return { ...EMPTY, ...resp }; + }), + release: vi.fn(), + _calls: calls, + }; + return client; +} + +const AGENT_URL = 'https://agent.example.com'; + +function makeRunRow(triggeredBy: string) { + return { + id: 'run-001', + agent_url: AGENT_URL, + lifecycle_stage: 'production', + overall_status: 'passing', + headline: null, + total_duration_ms: 100, + tested_at: new Date(), + tracks_json: [], + tracks_passed: 1, + tracks_failed: 0, + tracks_skipped: 0, + tracks_partial: 0, + agent_profile_json: null, + observations_json: null, + triggered_by: triggeredBy, + dry_run: false, + }; +} + +const minimalInput = (triggeredBy: 'heartbeat' | 'owner_test') => ({ + agent_url: AGENT_URL, + lifecycle_stage: 'production' as const, + overall_status: 'passing' as const, + tracks_json: [{ track: 'core', status: 'pass' as const, scenario_count: 1, passed_count: 1, duration_ms: 100 }], + tracks_passed: 1, + tracks_failed: 0, + tracks_skipped: 0, + tracks_partial: 0, + triggered_by: triggeredBy, + dry_run: false, +}); + +describe('ComplianceDatabase — last-write-wins on agent_compliance_status', () => { + let db: ComplianceDatabase; + + beforeEach(() => { + db = new ComplianceDatabase(); + vi.clearAllMocks(); + }); + + /** + * Contract: agent_compliance_status uses ON CONFLICT DO UPDATE (not DO NOTHING). + * Every recordComplianceRun call — regardless of triggered_by — overwrites the + * materialized status row. A future change to "pick highest-priority source" or + * "first-write-wins" would break this test. + */ + it('always upserts status regardless of triggered_by — last-write-wins', async () => { + const statusRow = { rows: [{ status: 'passing', previous_status: null }] }; + + const client = makeTransactionClient([ + EMPTY, // BEGIN + { rows: [makeRunRow('heartbeat')] }, // INSERT agent_compliance_runs + statusRow, // UPSERT agent_compliance_status + EMPTY, // COMMIT + ]); + mockedGetClient.mockResolvedValueOnce(client as any); + + await db.recordComplianceRun(minimalInput('heartbeat')); + + const upsertCall = client.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('ON CONFLICT (agent_url) DO UPDATE'), + ); + expect(upsertCall).toBeDefined(); + }); + + it('owner_test write at T+1 wins over prior heartbeat — triggered_by is forwarded verbatim', async () => { + const statusRow = { rows: [{ status: 'passing', previous_status: 'passing' }] }; + + const client1 = makeTransactionClient([ + EMPTY, + { rows: [makeRunRow('heartbeat')] }, + statusRow, + EMPTY, + ]); + mockedGetClient.mockResolvedValueOnce(client1 as any); + await db.recordComplianceRun(minimalInput('heartbeat')); + + const heartbeatRunInsert = client1.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('INSERT INTO agent_compliance_runs'), + ); + expect(heartbeatRunInsert).toBeDefined(); + expect(heartbeatRunInsert![1]).toContain('heartbeat'); + + const client2 = makeTransactionClient([ + EMPTY, + { rows: [makeRunRow('owner_test')] }, + statusRow, + EMPTY, + ]); + mockedGetClient.mockResolvedValueOnce(client2 as any); + await db.recordComplianceRun(minimalInput('owner_test')); + + const ownerTestRunInsert = client2.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('INSERT INTO agent_compliance_runs'), + ); + expect(ownerTestRunInsert).toBeDefined(); + expect(ownerTestRunInsert![1]).toContain('owner_test'); + + const ownerTestUpsert = client2.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('ON CONFLICT (agent_url) DO UPDATE'), + ); + expect(ownerTestUpsert).toBeDefined(); + }); + + it('heartbeat at T+3 wins over prior owner_test at T+2 — no source-priority filtering', async () => { + const statusRow = { rows: [{ status: 'passing', previous_status: 'passing' }] }; + + const client = makeTransactionClient([ + EMPTY, + { rows: [makeRunRow('heartbeat')] }, + statusRow, + EMPTY, + ]); + mockedGetClient.mockResolvedValueOnce(client as any); + await db.recordComplianceRun(minimalInput('heartbeat')); + + const runInsert = client.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('INSERT INTO agent_compliance_runs'), + ); + expect(runInsert![1]).toContain('heartbeat'); + + const upsert = client.query.mock.calls.find( + ([sql]: [string]) => typeof sql === 'string' && sql.includes('ON CONFLICT (agent_url) DO UPDATE'), + ); + expect(upsert).toBeDefined(); + }); + + it('getComplianceStatus LATERAL join returns last_triggered_by from most recent non-dry run', async () => { + const now = new Date(); + mockedQuery.mockResolvedValueOnce({ + rows: [{ + agent_url: AGENT_URL, + status: 'passing', + lifecycle_stage: 'production', + last_checked_at: now, + last_passed_at: now, + last_failed_at: null, + streak_days: 1, + streak_started_at: now, + tracks_summary_json: { core: 'pass' }, + headline: null, + previous_status: null, + status_changed_at: null, + updated_at: now, + last_triggered_by: 'owner_test', + }], + rowCount: 1, + command: '', + oid: 0, + fields: [], + }); + + const status = await db.getComplianceStatus(AGENT_URL); + + expect(status).not.toBeNull(); + expect(status!.last_triggered_by).toBe('owner_test'); + + const [sql] = mockedQuery.mock.calls[0]; + expect(sql).toContain('dry_run = false'); + expect(sql).toContain('ORDER BY tested_at DESC'); + expect(sql).toContain('LIMIT 1'); + }); +});