feat(registry): persist managerdomain discovery provenance on publisher rows

.changeset/publisher-discovery-provenance.md

@@ -0,0 +1,18 @@
+---
+---
+
+feat(registry): persist managerdomain discovery provenance on publisher rows
+
+Adds `discovery_method` and `manager_domain` columns to the `publishers`
+table (migration 470) and threads provenance through all crawler write
+paths (`cacheAdagentsManifest`, events). Both fields are now surfaced in
+the `/api/validate-publisher` response and `/api/registry/publisher`
+endpoint so callers can distinguish direct origin-attestation from
+one-hop ads.txt `MANAGERDOMAIN` delegation.
+
+Events `publisher.adagents_discovered` and `publisher.adagents_changed`
+now carry `discovery_method` and `manager_domain` in their payloads.
+
+Non-protocol change (server infrastructure only). Follow-up: reverse
+index + fan-out (issue #4200 item 2), per-agent `source` enum extension
+to `adagents_json_via_manager`.

server/src/crawler.ts

@@ -390,6 +390,8 @@ export class CrawlerService {
                 statusCode: validation.status_code,
                 responseBytes: validation.response_bytes,
                 resolvedUrl: validation.resolved_url,
+                discoveryMethod: validation.discovery_method,
+                managerDomain: validation.manager_domain,
               },
             );
 
@@ -464,6 +466,8 @@ export class CrawlerService {
                 statusCode: validation.status_code,
                 responseBytes: validation.response_bytes,
                 resolvedUrl: validation.resolved_url,
+                discoveryMethod: validation.discovery_method,
+                managerDomain: validation.manager_domain,
               },
             );
 
@@ -810,7 +814,7 @@ export class CrawlerService {
   private async cacheAdagentsManifest(
     domain: string,
     manifest: AdagentsManifest,
-    meta?: { statusCode?: number; responseBytes?: number; resolvedUrl?: string },
+    meta?: { statusCode?: number; responseBytes?: number; resolvedUrl?: string; discoveryMethod?: string; managerDomain?: string },
   ): Promise<void> {
     try {
       await this.publisherDb.upsertAdagentsCache({
@@ -819,6 +823,8 @@ export class CrawlerService {
         statusCode: meta?.statusCode,
         responseBytes: meta?.responseBytes,
         resolvedUrl: meta?.resolvedUrl,
+        discoveryMethod: meta?.discoveryMethod,
+        managerDomain: meta?.managerDomain,
       });
     } catch (err) {
       log.warn({ domain, err: err instanceof Error ? err.message : err }, 'Publisher cache write failed');
@@ -1101,6 +1107,8 @@ export class CrawlerService {
           statusCode: validation.status_code,
           responseBytes: validation.response_bytes,
           resolvedUrl: validation.resolved_url,
+          discoveryMethod: validation.discovery_method,
+          managerDomain: validation.manager_domain,
         },
       );
 
@@ -1135,6 +1143,8 @@ export class CrawlerService {
             publisher_domain: domain,
             agent_count: validation.raw_data.authorized_agents.length,
             property_count: validation.raw_data.properties?.length ?? 0,
+            discovery_method: validation.discovery_method,
+            manager_domain: validation.manager_domain,
           },
           actor: 'api:crawl-request',
         });
@@ -1286,6 +1296,8 @@ export class CrawlerService {
         statusCode: validation.status_code,
         responseBytes: validation.response_bytes,
         resolvedUrl: validation.resolved_url,
+        discoveryMethod: validation.discovery_method,
+        managerDomain: validation.manager_domain,
       },
     );
 
@@ -1319,6 +1331,8 @@ export class CrawlerService {
           agent_count: validation.raw_data.authorized_agents.length,
           property_count: validation.raw_data.properties?.length ?? 0,
           source: 'catalog_crawl',
+          discovery_method: validation.discovery_method,
+          manager_domain: validation.manager_domain,
         },
         actor: 'pipeline:catalog_crawl',
       });

server/src/db/migrations/470_publisher_discovery_provenance.sql

@@ -0,0 +1,41 @@
+-- Persist discovery provenance fields from AdAgentsValidationResult onto the
+-- publishers overlay row so the AAO API can surface how authorization was
+-- discovered (direct vs. authoritative_location vs. ads_txt_managerdomain)
+-- and, when a managerdomain hop was used, which manager domain served the
+-- manifest.
+--
+-- The CHECK constraint mirrors the DiscoveryMethod type in
+-- server/src/adagents-manager.ts so the DB rejects invalid values at
+-- write time rather than silently storing garbage.
+
+ALTER TABLE publishers
+  ADD COLUMN discovery_method TEXT
+    CHECK (discovery_method IN ('direct', 'authoritative_location', 'ads_txt_managerdomain')),
+  ADD COLUMN manager_domain TEXT;
+
+-- Backfill: every successfully-validated publisher row that exists at the
+-- time of this migration was necessarily discovered via the direct path —
+-- the other two methods didn't exist before 4173/4204 landed. Stamping
+-- 'direct' on those rows eliminates the otherwise-confusing NULL window
+-- between this migration and the next crawl cycle, so /api/validate-publisher
+-- and /api/registry/publisher return a stable provenance value immediately.
+UPDATE publishers
+   SET discovery_method = 'direct'
+ WHERE discovery_method IS NULL
+   AND source_type = 'adagents_json'
+   AND adagents_json IS NOT NULL;
+
+-- Partial index supports the manager → publishers reverse lookup planned
+-- in #4200 item 2 (queue-backed fan-out when a manager rotates their
+-- adagents.json). Building it here is essentially free — the column is
+-- mostly NULL, so the index footprint is tiny — and means item 2 ships
+-- without another schema migration.
+CREATE INDEX idx_publishers_manager_domain
+  ON publishers (manager_domain)
+  WHERE manager_domain IS NOT NULL;
+
+COMMENT ON COLUMN publishers.discovery_method IS
+  'How the publisher''s adagents.json was discovered on the most recent successful crawl. ''direct'': publisher''s own /.well-known/ served the document. ''authoritative_location'': publisher''s stub redirected to a third-party canonical URL. ''ads_txt_managerdomain'': discovery fell back to a manager domain via ads.txt MANAGERDOMAIN delegation. Backfilled to ''direct'' for previously-validated rows.';
+
+COMMENT ON COLUMN publishers.manager_domain IS
+  'The manager domain whose adagents.json was used to authorize this publisher''s agents. Non-NULL only when discovery_method = ''ads_txt_managerdomain''. Matches the MANAGERDOMAIN value from the publisher''s ads.txt. NULL for all other discovery methods.';

server/src/db/publisher-db.ts

@@ -77,6 +77,17 @@ export interface UpsertAdagentsCacheInput {
    * `self_redirected` or `aao_hosted`.
    */
   resolvedUrl?: string;
+  /**
+   * How the publisher's adagents.json was discovered. Mirrors
+   * AdAgentsValidationResult.discovery_method. Written to
+   * publishers.discovery_method so the API can surface provenance.
+   */
+  discoveryMethod?: string;
+  /**
+   * When discoveryMethod is 'ads_txt_managerdomain', the manager domain
+   * whose adagents.json was used. Written to publishers.manager_domain.
+   */
+  managerDomain?: string;
 }
 
 const ADAGENTS_CREATED_BY_PREFIX = 'adagents_json:';
@@ -229,8 +240,9 @@ export class PublisherDatabase {
       await client.query(
         `INSERT INTO publishers
            (domain, adagents_json, source_type, last_validated, expires_at,
-            last_http_status, last_response_bytes, resolved_url)
-         VALUES ($1, $2::jsonb, 'adagents_json', NOW(), $3, $4, $5, $6)
+            last_http_status, last_response_bytes, resolved_url,
+            discovery_method, manager_domain)
+         VALUES ($1, $2::jsonb, 'adagents_json', NOW(), $3, $4, $5, $6, $7, $8)
          ON CONFLICT (domain) DO UPDATE SET
            adagents_json = EXCLUDED.adagents_json,
            source_type = 'adagents_json',
@@ -239,6 +251,8 @@ export class PublisherDatabase {
            last_http_status = EXCLUDED.last_http_status,
            last_response_bytes = EXCLUDED.last_response_bytes,
            resolved_url = EXCLUDED.resolved_url,
+           discovery_method = EXCLUDED.discovery_method,
+           manager_domain = EXCLUDED.manager_domain,
            updated_at = NOW()`,
         [
           domain,
@@ -247,6 +261,8 @@ export class PublisherDatabase {
           clampHttpStatus(input.statusCode),
           input.responseBytes ?? null,
           truncateResolvedUrl(input.resolvedUrl),
+          input.discoveryMethod ?? null,
+          input.managerDomain ?? null,
         ]
       );
 

server/src/http.ts

@@ -8872,6 +8872,8 @@ ${p.category ? `<category>${p.category}</category>\n` : ''}<url>${publishedUrl}<
           valid: result.valid,
           domain: result.domain,
           url: result.url,
+          discovery_method: result.discovery_method,
+          manager_domain: result.manager_domain ?? undefined,
           agent_count: stats.agentCount,
           property_count: stats.propertyCount,
           property_type_counts: stats.propertyTypeCounts,

server/src/routes/registry-api.ts

@@ -1045,6 +1045,12 @@ registry.registerPath({
             valid: z.boolean(),
             domain: z.string(),
             url: z.string().optional(),
+            discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).openapi({
+              description: "How the publisher's adagents.json was discovered. `ads_txt_managerdomain` indicates one-hop delegation via ads.txt MANAGERDOMAIN.",
+            }),
+            manager_domain: z.string().optional().openapi({
+              description: "Manager domain that served the manifest. Present only when discovery_method is ads_txt_managerdomain.",
+            }),
             agent_count: z.number().int(),
             property_count: z.number().int(),
             property_type_counts: z.record(z.string(), z.number().int()),
@@ -5895,14 +5901,17 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router {
           last_http_status: number | null;
           last_response_bytes: number | null;
           resolved_url: string | null;
+          discovery_method: string | null;
+          manager_domain: string | null;
         }>(
           // Drop the source_type='adagents_json' filter. Phase B writes
           // failed-fetch metadata onto rows with source_type='community',
           // and we want the verifier UI to surface
           // "Last attempted: <ts> · HTTP <code>" even for never-validated
           // domains. Read whatever row exists; downstream code handles
           // null adagents_json gracefully.
-          `SELECT adagents_json, last_validated, last_http_status, last_response_bytes, resolved_url
+          `SELECT adagents_json, last_validated, last_http_status, last_response_bytes, resolved_url,
+                  discovery_method, manager_domain
              FROM publishers WHERE domain = $1 LIMIT 1`,
           [domain],
         ).then(r => r.rows[0] ?? null),
@@ -5912,6 +5921,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router {
       const cachedHttpStatus = cachedAdagentsRow?.last_http_status ?? null;
       const cachedResponseBytes = cachedAdagentsRow?.last_response_bytes ?? null;
       const cachedResolvedUrl = cachedAdagentsRow?.resolved_url ?? null;
+      const cachedDiscoveryMethod = cachedAdagentsRow?.discovery_method ?? null;
+      const cachedManagerDomain = cachedAdagentsRow?.manager_domain ?? null;
 
       // Auto-crawl on view: if we've never crawled this domain (adagents
       // never seen, brand never seen), kick off background fetches so a
@@ -6306,6 +6317,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router {
         domain,
         member,
         adagents_valid: adagentsValid,
+        discovery_method: cachedDiscoveryMethod ?? undefined,
+        manager_domain: cachedManagerDomain ?? undefined,
         hosting,
         files,
         properties: projectedProperties,
@@ -6733,6 +6746,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router {
         valid: result.valid,
         domain: result.domain,
         url: result.url,
+        discovery_method: result.discovery_method,
+        manager_domain: result.manager_domain ?? undefined,
         agent_count: stats.agentCount,
         property_count: stats.propertyCount,
         property_type_counts: stats.propertyTypeCounts,

server/src/schemas/registry.ts

@@ -651,6 +651,14 @@ export const PublisherLookupResultSchema = z
     domain: z.string().openapi({ example: "voxmedia.com" }),
     member: MemberRefSchema.nullable(),
     adagents_valid: z.boolean().nullable(),
+    discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).nullable().optional().openapi({
+      description:
+        "How the publisher's adagents.json was discovered on the most recent successful crawl. `direct`: publisher's own /.well-known/ served the document. `authoritative_location`: publisher's stub redirected to a canonical URL. `ads_txt_managerdomain`: manifest was discovered via ads.txt MANAGERDOMAIN delegation — see `manager_domain` for which manager served it. Null until first crawl after migration 470.",
+    }),
+    manager_domain: z.string().nullable().optional().openapi({
+      description:
+        "The manager domain whose adagents.json was used to authorize this publisher's agents. Non-null only when `discovery_method` is `ads_txt_managerdomain`. Matches the MANAGERDOMAIN value from the publisher's ads.txt.",
+    }),
     hosting: PublisherHostingSchema,
     files: PublisherFilesSchema.optional().openapi({
       description: