From 48f96aa36f92e45256cf8071ece45fe6f226f6f7 Mon Sep 17 00:00:00 2001 From: Emma Mulitz Date: Fri, 8 May 2026 16:18:57 -0400 Subject: [PATCH] feat(dashboard): surface verdict_source + per-run triggered_by badge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 2 of the #4247 unification stack. Reads two fields PR #4250 added to the compliance API but the dashboard wasn't yet rendering: - compliance tile: appends "(your test)" / "(heartbeat)" / "(manual)" / "(webhook)" after Last checked, so operators see whether the current verdict came from their own evaluate_agent_quality run or the scheduled heartbeat. - history panel: per-run badge with the same source label, info-blue for owner_test and neutral for the rest. Pre-PR-1 rows render with neutral — no regression. No backend changes; pure UI surfacing of fields already in the API. Stacked on PR #4250. --- .../dashboard-surfaces-verdict-source.md | 31 +++++++++++++++++++ server/public/dashboard-agents.html | 31 ++++++++++++++++++- server/src/routes/registry-api.ts | 21 ++++++++----- server/src/schemas/registry.ts | 2 ++ server/src/services/membership-tiers.ts | 11 +++++++ server/tests/unit/membership-tiers.test.ts | 2 ++ 6 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 .changeset/dashboard-surfaces-verdict-source.md diff --git a/.changeset/dashboard-surfaces-verdict-source.md b/.changeset/dashboard-surfaces-verdict-source.md new file mode 100644 index 0000000000..e7fa44366d --- /dev/null +++ b/.changeset/dashboard-surfaces-verdict-source.md @@ -0,0 +1,31 @@ +--- +--- + +Dashboard `/dashboard/agents` surfaces the new `verdict_source` field on the +compliance tile and a per-run "Your test / Heartbeat / Manual / Webhook" +badge in the History panel. PR 2 of the #4247 unification stack — +read-side cleanup that lets owners distinguish their own on-demand +runs from scheduled heartbeat verdicts at a glance. + +**Context.** PR #4250 added `verdict_source` to +`/api/registry/agents/:url/compliance` and `triggered_by` to each row +returned by `/api/registry/agents/:url/compliance/history`. Both fields +were unrendered in the dashboard until this PR. + +**What changes.** + +- Compliance tile shows `Last checked: 3m ago (your test)` / + `(heartbeat)` / `(manual)` / `(webhook)` after the timestamp. Empty + string when `verdict_source` is null (never run). +- History panel renders a colored badge per run row: + - `Your test` (info-blue) for `triggered_by = 'owner_test'` + - `Heartbeat` (neutral) for `triggered_by = 'heartbeat'` + - `Manual` / `Webhook` (neutral) for the other enum values + +No backend changes; this is pure UI surfacing of fields the API already +emits. Pre-PR-1 rows (which only have `'heartbeat'` / `'manual'` / +`'webhook'`) render with the neutral badge — no regression. + +**Out of scope** (PR 3 of #4247): dropping `agent_test_history` and +backfilling owner-triggered rows. Tracked separately so the destructive +migration soaks behind the read-only UI change. diff --git a/server/public/dashboard-agents.html b/server/public/dashboard-agents.html index 810869c513..856fab7e9d 100644 --- a/server/public/dashboard-agents.html +++ b/server/public/dashboard-agents.html @@ -1501,6 +1501,19 @@

Agents

? timeAgo(new Date(cs.last_checked_at)) : 'never'; + // Surface the verdict source so the operator knows whether the + // current status came from the scheduled heartbeat or their own + // owner-triggered test run. PR #4250 populates cs.verdict_source + // ('heartbeat' | 'owner_test' | 'manual' | 'webhook' | null when + // never run). Displayed inline with "Last checked" so the + // semantic shift on the public compliance contract is visible + // to the operator without having to read the changelog. + const verdictSourceLabel = cs.verdict_source === 'owner_test' ? ' (your test)' + : cs.verdict_source === 'heartbeat' ? ' (heartbeat)' + : cs.verdict_source === 'manual' ? ' (manual)' + : cs.verdict_source === 'webhook' ? ' (webhook)' + : ''; + const isPublic = cs.status !== 'opted_out'; return ` @@ -1527,7 +1540,7 @@

Agents

${visibilitySelectorHtml}
- Last checked: ${escapeHtml(lastChecked)} + Last checked: ${escapeHtml(lastChecked)}${escapeHtml(verdictSourceLabel)}
'; if (runTracks) { html += '
' + runTracks + '
'; diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index a35e8a055c..ad3bc2cc0a 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -4332,14 +4332,19 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { membership_tier_label: ownerMembership.membership_tier_label, subscription_status: ownerMembership.subscription_status, is_api_access_tier: ownerMembership.is_api_access_tier, - // `triggered_by` is retained as internal audit on agent_compliance_runs - // but deliberately not exposed publicly: heartbeat and owner_test both - // call comply() against the same registered URL with the same - // owner-saved credentials; the verdict's truth content is identical - // regardless of who pulled the trigger. Exposing the source label - // creates a buyer-facing trust distinction that the underlying - // observation doesn't actually carry. Internal dashboards may still - // surface triggered_by as a UX cue (see #4263). + // `verdict_source` is owner-scoped: operators benefit from seeing + // whether the current verdict came from their own owner_test vs + // the scheduled heartbeat (UX cue while iterating on a fix). Non- + // owners see null — heartbeat and owner_test both call comply() + // against the same registered URL with the same owner-saved + // credentials, so exposing the source label publicly would + // create a trust distinction the underlying observation doesn't + // actually carry. Gated on `is_owner` (any owner, including free + // tier) — `is_api_access_tier` would be too narrow and would + // hide the UX cue from Explorer-tier agent owners. + verdict_source: ownerMembership.is_owner + ? (status.last_triggered_by ?? null) + : null, verified: badges.length > 0, verified_badges: badges.map(b => ({ role: b.role, diff --git a/server/src/schemas/registry.ts b/server/src/schemas/registry.ts index 4a59bdd2be..e243a98683 100644 --- a/server/src/schemas/registry.ts +++ b/server/src/schemas/registry.ts @@ -341,6 +341,8 @@ export const AgentComplianceDetailSchema = z membership_tier_label: z.string().nullable().optional().openapi({ description: "Owner-scoped: human-readable label for membership_tier (e.g. 'Builder'). Null for non-owners." }), subscription_status: z.string().nullable().optional().openapi({ description: "Owner-scoped: the agent owner's subscription status (active, past_due, trialing, etc.). Null for non-owners." }), is_api_access_tier: z.boolean().optional().openapi({ description: "Owner-scoped: true when the owner's tier and subscription status grant badge eligibility. False for non-owners. Single source of truth — UI should not re-derive." }), + verdict_source: z.enum(["heartbeat", "owner_test", "manual", "webhook"]).nullable().optional() + .openapi({ description: "Owner-scoped: triggered_by value of the most recent non-dry-run compliance check. Null for non-owners and when no run has been recorded. Operators use this as a UX cue ('did this verdict come from my recent test or the system heartbeat?')." }), verified: z.boolean().optional(), verified_badges: z.array(VerificationBadgeSchema).optional(), }) diff --git a/server/src/services/membership-tiers.ts b/server/src/services/membership-tiers.ts index b2ece6fb72..3c4595deee 100644 --- a/server/src/services/membership-tiers.ts +++ b/server/src/services/membership-tiers.ts @@ -65,6 +65,15 @@ export function tierLabel(tier: string | null | undefined): string | null { } export interface OwnerMembership { + /** + * True only when the caller actually owns the agent (the resolver found + * a matching member_profiles row + organization_memberships row for them). + * Use this for owner-only feature gates that don't need a specific tier — + * `is_api_access_tier` is narrower (only premium tiers + active subs). + * Not surfaced in API responses; internal struct only. Response shape + * stays owner-detect-resistant via constant keys + null values. + */ + is_owner: boolean; membership_tier: string | null; membership_tier_label: string | null; subscription_status: string | null; @@ -72,6 +81,7 @@ export interface OwnerMembership { } const EMPTY_OWNER_MEMBERSHIP: OwnerMembership = { + is_owner: false, membership_tier: null, membership_tier_label: null, subscription_status: null, @@ -114,6 +124,7 @@ export async function resolveOwnerMembership( const tier = orgRow.membership_tier ?? null; const subStatus = orgRow.subscription_status ?? null; return { + is_owner: true, membership_tier: tier, membership_tier_label: tierLabel(tier), subscription_status: subStatus, diff --git a/server/tests/unit/membership-tiers.test.ts b/server/tests/unit/membership-tiers.test.ts index 4ecb198dde..588d413883 100644 --- a/server/tests/unit/membership-tiers.test.ts +++ b/server/tests/unit/membership-tiers.test.ts @@ -92,6 +92,7 @@ describe('membership-tiers', () => { describe('resolveOwnerMembership — security boundary', () => { const EMPTY = { + is_owner: false, membership_tier: null, membership_tier_label: null, subscription_status: null, @@ -126,6 +127,7 @@ describe('membership-tiers', () => { fetchOrgMembership: async () => ({ membership_tier: 'company_standard', subscription_status: 'active' }), }); expect(result).toEqual({ + is_owner: true, membership_tier: 'company_standard', membership_tier_label: 'Builder', subscription_status: 'active',