From 745d205d861d7b7f52b5db089ba7b9e352a68546 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 7 May 2026 19:10:03 +0000 Subject: [PATCH 1/4] feat(registry): persist managerdomain discovery provenance on publisher rows Threads discovery_method + manager_domain from AdAgentsValidationResult (already populated by PR #4173) through all crawler write paths, events, and API endpoints for issue #4200 items 1, 3, 4. - Migration 470: ADD COLUMN discovery_method TEXT (CHECK constraint) + manager_domain TEXT to publishers table - UpsertAdagentsCacheInput extended; SQL upsert writes both columns - All 3 cacheAdagentsManifest call sites pass provenance fields - publisher.adagents_changed + publisher.adagents_discovered events now carry discovery_method + manager_domain in their payloads - /api/validate-publisher (http.ts) and /api/public/validate-publisher (registry-api.ts) responses include both fields - /api/registry/publisher reads discovery_method + manager_domain from the publishers row and includes them at the publisher level - PublisherLookupResultSchema and /api/public/validate-publisher OpenAPI schema updated to declare the new fields Non-breaking: all columns nullable, all response fields optional additive. Item 2 (reverse index + queue-backed fan-out) is a follow-up. Refs #4200 https://claude.ai/code/cse_01SX6coXKTUA3sLPB6dfcjZt --- .changeset/publisher-discovery-provenance.md | 18 +++++++++++++ server/src/crawler.ts | 16 +++++++++++- .../470_publisher_discovery_provenance.sql | 26 +++++++++++++++++++ server/src/db/publisher-db.ts | 20 ++++++++++++-- server/src/http.ts | 2 ++ server/src/routes/registry-api.ts | 17 +++++++++++- server/src/schemas/registry.ts | 8 ++++++ 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 .changeset/publisher-discovery-provenance.md create mode 100644 server/src/db/migrations/470_publisher_discovery_provenance.sql diff --git a/.changeset/publisher-discovery-provenance.md b/.changeset/publisher-discovery-provenance.md new file mode 100644 index 0000000000..b273ad21be --- /dev/null +++ b/.changeset/publisher-discovery-provenance.md @@ -0,0 +1,18 @@ +--- +--- + +feat(registry): persist managerdomain discovery provenance on publisher rows + +Adds `discovery_method` and `manager_domain` columns to the `publishers` +table (migration 470) and threads provenance through all crawler write +paths (`cacheAdagentsManifest`, events). Both fields are now surfaced in +the `/api/validate-publisher` response and `/api/registry/publisher` +endpoint so callers can distinguish direct origin-attestation from +one-hop ads.txt `MANAGERDOMAIN` delegation. + +Events `publisher.adagents_discovered` and `publisher.adagents_changed` +now carry `discovery_method` and `manager_domain` in their payloads. + +Non-protocol change (server infrastructure only). Follow-up: reverse +index + fan-out (issue #4200 item 2), per-agent `source` enum extension +to `adagents_json_via_manager`. diff --git a/server/src/crawler.ts b/server/src/crawler.ts index dd0bec74fb..d19547ce7b 100644 --- a/server/src/crawler.ts +++ b/server/src/crawler.ts @@ -390,6 +390,8 @@ export class CrawlerService { statusCode: validation.status_code, responseBytes: validation.response_bytes, resolvedUrl: validation.resolved_url, + discoveryMethod: validation.discovery_method, + managerDomain: validation.manager_domain, }, ); @@ -464,6 +466,8 @@ export class CrawlerService { statusCode: validation.status_code, responseBytes: validation.response_bytes, resolvedUrl: validation.resolved_url, + discoveryMethod: validation.discovery_method, + managerDomain: validation.manager_domain, }, ); @@ -810,7 +814,7 @@ export class CrawlerService { private async cacheAdagentsManifest( domain: string, manifest: AdagentsManifest, - meta?: { statusCode?: number; responseBytes?: number; resolvedUrl?: string }, + meta?: { statusCode?: number; responseBytes?: number; resolvedUrl?: string; discoveryMethod?: string; managerDomain?: string }, ): Promise { try { await this.publisherDb.upsertAdagentsCache({ @@ -819,6 +823,8 @@ export class CrawlerService { statusCode: meta?.statusCode, responseBytes: meta?.responseBytes, resolvedUrl: meta?.resolvedUrl, + discoveryMethod: meta?.discoveryMethod, + managerDomain: meta?.managerDomain, }); } catch (err) { log.warn({ domain, err: err instanceof Error ? err.message : err }, 'Publisher cache write failed'); @@ -1101,6 +1107,8 @@ export class CrawlerService { statusCode: validation.status_code, responseBytes: validation.response_bytes, resolvedUrl: validation.resolved_url, + discoveryMethod: validation.discovery_method, + managerDomain: validation.manager_domain, }, ); @@ -1135,6 +1143,8 @@ export class CrawlerService { publisher_domain: domain, agent_count: validation.raw_data.authorized_agents.length, property_count: validation.raw_data.properties?.length ?? 0, + discovery_method: validation.discovery_method, + manager_domain: validation.manager_domain ?? undefined, }, actor: 'api:crawl-request', }); @@ -1286,6 +1296,8 @@ export class CrawlerService { statusCode: validation.status_code, responseBytes: validation.response_bytes, resolvedUrl: validation.resolved_url, + discoveryMethod: validation.discovery_method, + managerDomain: validation.manager_domain, }, ); @@ -1319,6 +1331,8 @@ export class CrawlerService { agent_count: validation.raw_data.authorized_agents.length, property_count: validation.raw_data.properties?.length ?? 0, source: 'catalog_crawl', + discovery_method: validation.discovery_method, + manager_domain: validation.manager_domain ?? undefined, }, actor: 'pipeline:catalog_crawl', }); diff --git a/server/src/db/migrations/470_publisher_discovery_provenance.sql b/server/src/db/migrations/470_publisher_discovery_provenance.sql new file mode 100644 index 0000000000..7a720518f0 --- /dev/null +++ b/server/src/db/migrations/470_publisher_discovery_provenance.sql @@ -0,0 +1,26 @@ +-- Persist discovery provenance fields from AdAgentsValidationResult onto the +-- publishers overlay row so the AAO API can surface how authorization was +-- discovered (direct vs. authoritative_location vs. ads_txt_managerdomain) +-- and, when a managerdomain hop was used, which manager domain served the +-- manifest. +-- +-- These columns are set by the crawler on every successful manifest cache +-- write. Backfill is implicit: every publisher is re-crawled within the +-- 60-minute cadence, so the new columns populate naturally on next fetch. +-- Callers that see NULL can treat it as "not yet re-crawled since this +-- migration" and fall back to existing adagents_valid signal. +-- +-- The CHECK constraint mirrors the DiscoveryMethod type in +-- server/src/adagents-manager.ts so the DB rejects invalid values at +-- write time rather than silently storing garbage. + +ALTER TABLE publishers + ADD COLUMN discovery_method TEXT + CHECK (discovery_method IN ('direct', 'authoritative_location', 'ads_txt_managerdomain')), + ADD COLUMN manager_domain TEXT; + +COMMENT ON COLUMN publishers.discovery_method IS + 'How the publisher''s adagents.json was discovered on the most recent successful crawl. ''direct'': publisher''s own /.well-known/ served the document. ''authoritative_location'': publisher''s stub redirected to a third-party canonical URL. ''ads_txt_managerdomain'': discovery fell back to a manager domain via ads.txt MANAGERDOMAIN delegation. NULL until first successful crawl after migration 470.'; + +COMMENT ON COLUMN publishers.manager_domain IS + 'The manager domain whose adagents.json was used to authorize this publisher''s agents. Non-NULL only when discovery_method = ''ads_txt_managerdomain''. Matches the MANAGERDOMAIN value from the publisher''s ads.txt. NULL for all other discovery methods.'; diff --git a/server/src/db/publisher-db.ts b/server/src/db/publisher-db.ts index f0bb45a970..c57c6ab449 100644 --- a/server/src/db/publisher-db.ts +++ b/server/src/db/publisher-db.ts @@ -77,6 +77,17 @@ export interface UpsertAdagentsCacheInput { * `self_redirected` or `aao_hosted`. */ resolvedUrl?: string; + /** + * How the publisher's adagents.json was discovered. Mirrors + * AdAgentsValidationResult.discovery_method. Written to + * publishers.discovery_method so the API can surface provenance. + */ + discoveryMethod?: string; + /** + * When discoveryMethod is 'ads_txt_managerdomain', the manager domain + * whose adagents.json was used. Written to publishers.manager_domain. + */ + managerDomain?: string; } const ADAGENTS_CREATED_BY_PREFIX = 'adagents_json:'; @@ -229,8 +240,9 @@ export class PublisherDatabase { await client.query( `INSERT INTO publishers (domain, adagents_json, source_type, last_validated, expires_at, - last_http_status, last_response_bytes, resolved_url) - VALUES ($1, $2::jsonb, 'adagents_json', NOW(), $3, $4, $5, $6) + last_http_status, last_response_bytes, resolved_url, + discovery_method, manager_domain) + VALUES ($1, $2::jsonb, 'adagents_json', NOW(), $3, $4, $5, $6, $7, $8) ON CONFLICT (domain) DO UPDATE SET adagents_json = EXCLUDED.adagents_json, source_type = 'adagents_json', @@ -239,6 +251,8 @@ export class PublisherDatabase { last_http_status = EXCLUDED.last_http_status, last_response_bytes = EXCLUDED.last_response_bytes, resolved_url = EXCLUDED.resolved_url, + discovery_method = EXCLUDED.discovery_method, + manager_domain = EXCLUDED.manager_domain, updated_at = NOW()`, [ domain, @@ -247,6 +261,8 @@ export class PublisherDatabase { clampHttpStatus(input.statusCode), input.responseBytes ?? null, truncateResolvedUrl(input.resolvedUrl), + input.discoveryMethod ?? null, + input.managerDomain ?? null, ] ); diff --git a/server/src/http.ts b/server/src/http.ts index 9e5a764518..e61988b7b3 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -8872,6 +8872,8 @@ ${p.category ? `${p.category}\n` : ''}${publishedUrl}< valid: result.valid, domain: result.domain, url: result.url, + discovery_method: result.discovery_method, + manager_domain: result.manager_domain ?? undefined, agent_count: stats.agentCount, property_count: stats.propertyCount, property_type_counts: stats.propertyTypeCounts, diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 22836ee2fa..cb127bafde 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -1045,6 +1045,12 @@ registry.registerPath({ valid: z.boolean(), domain: z.string(), url: z.string().optional(), + discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).optional().openapi({ + description: "How the publisher's adagents.json was discovered. `ads_txt_managerdomain` indicates one-hop delegation via ads.txt MANAGERDOMAIN.", + }), + manager_domain: z.string().optional().openapi({ + description: "Manager domain that served the manifest. Present only when discovery_method is ads_txt_managerdomain.", + }), agent_count: z.number().int(), property_count: z.number().int(), property_type_counts: z.record(z.string(), z.number().int()), @@ -5895,6 +5901,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { last_http_status: number | null; last_response_bytes: number | null; resolved_url: string | null; + discovery_method: string | null; + manager_domain: string | null; }>( // Drop the source_type='adagents_json' filter. Phase B writes // failed-fetch metadata onto rows with source_type='community', @@ -5902,7 +5910,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { // "Last attempted: · HTTP " even for never-validated // domains. Read whatever row exists; downstream code handles // null adagents_json gracefully. - `SELECT adagents_json, last_validated, last_http_status, last_response_bytes, resolved_url + `SELECT adagents_json, last_validated, last_http_status, last_response_bytes, resolved_url, + discovery_method, manager_domain FROM publishers WHERE domain = $1 LIMIT 1`, [domain], ).then(r => r.rows[0] ?? null), @@ -5912,6 +5921,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { const cachedHttpStatus = cachedAdagentsRow?.last_http_status ?? null; const cachedResponseBytes = cachedAdagentsRow?.last_response_bytes ?? null; const cachedResolvedUrl = cachedAdagentsRow?.resolved_url ?? null; + const cachedDiscoveryMethod = cachedAdagentsRow?.discovery_method ?? null; + const cachedManagerDomain = cachedAdagentsRow?.manager_domain ?? null; // Auto-crawl on view: if we've never crawled this domain (adagents // never seen, brand never seen), kick off background fetches so a @@ -6306,6 +6317,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { domain, member, adagents_valid: adagentsValid, + discovery_method: cachedDiscoveryMethod, + manager_domain: cachedManagerDomain, hosting, files, properties: projectedProperties, @@ -6733,6 +6746,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { valid: result.valid, domain: result.domain, url: result.url, + discovery_method: result.discovery_method, + manager_domain: result.manager_domain ?? undefined, agent_count: stats.agentCount, property_count: stats.propertyCount, property_type_counts: stats.propertyTypeCounts, diff --git a/server/src/schemas/registry.ts b/server/src/schemas/registry.ts index 41aeef27a1..4a59bdd2be 100644 --- a/server/src/schemas/registry.ts +++ b/server/src/schemas/registry.ts @@ -651,6 +651,14 @@ export const PublisherLookupResultSchema = z domain: z.string().openapi({ example: "voxmedia.com" }), member: MemberRefSchema.nullable(), adagents_valid: z.boolean().nullable(), + discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).nullable().optional().openapi({ + description: + "How the publisher's adagents.json was discovered on the most recent successful crawl. `direct`: publisher's own /.well-known/ served the document. `authoritative_location`: publisher's stub redirected to a canonical URL. `ads_txt_managerdomain`: manifest was discovered via ads.txt MANAGERDOMAIN delegation — see `manager_domain` for which manager served it. Null until first crawl after migration 470.", + }), + manager_domain: z.string().nullable().optional().openapi({ + description: + "The manager domain whose adagents.json was used to authorize this publisher's agents. Non-null only when `discovery_method` is `ads_txt_managerdomain`. Matches the MANAGERDOMAIN value from the publisher's ads.txt.", + }), hosting: PublisherHostingSchema, files: PublisherFilesSchema.optional().openapi({ description: From e941ac8c7d8870c279bb714416f42f7a0c465563 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 7 May 2026 19:10:42 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix(registry):=20null=20=E2=86=92=20undefin?= =?UTF-8?q?ed=20for=20absent=20discovery=20fields=20in=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /api/registry/publisher: discovery_method and manager_domain now use ?? undefined so absent values omit the key from JSON rather than serialising null - Event payloads: remove ?? undefined no-op (manager_domain is already string | undefined on AdAgentsValidationResult) https://claude.ai/code/cse_01SX6coXKTUA3sLPB6dfcjZt --- server/src/crawler.ts | 4 ++-- server/src/routes/registry-api.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/crawler.ts b/server/src/crawler.ts index d19547ce7b..cf180e0d6b 100644 --- a/server/src/crawler.ts +++ b/server/src/crawler.ts @@ -1144,7 +1144,7 @@ export class CrawlerService { agent_count: validation.raw_data.authorized_agents.length, property_count: validation.raw_data.properties?.length ?? 0, discovery_method: validation.discovery_method, - manager_domain: validation.manager_domain ?? undefined, + manager_domain: validation.manager_domain, }, actor: 'api:crawl-request', }); @@ -1332,7 +1332,7 @@ export class CrawlerService { property_count: validation.raw_data.properties?.length ?? 0, source: 'catalog_crawl', discovery_method: validation.discovery_method, - manager_domain: validation.manager_domain ?? undefined, + manager_domain: validation.manager_domain, }, actor: 'pipeline:catalog_crawl', }); diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index cb127bafde..2d94e566f7 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -6317,8 +6317,8 @@ export function createRegistryApiRouter(config: RegistryApiConfig): Router { domain, member, adagents_valid: adagentsValid, - discovery_method: cachedDiscoveryMethod, - manager_domain: cachedManagerDomain, + discovery_method: cachedDiscoveryMethod ?? undefined, + manager_domain: cachedManagerDomain ?? undefined, hosting, files, properties: projectedProperties, From 167396a7416270f18ad68860387f81cbc1838f01 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 7 May 2026 17:46:20 -0400 Subject: [PATCH 3/4] chore(adagents): tighten provenance schema and prep for reverse-index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small improvements based on expert review of #4204 before merge: 1. **Backfill `discovery_method = 'direct'` for previously-validated rows.** Pre-migration, every successfully-validated publisher was necessarily discovered via the direct path — the other two methods didn't exist before #4173/#4204 landed. Stamping `direct` on those rows eliminates the otherwise-confusing NULL window between this migration and the next 60-minute crawl cycle, so the API returns a stable provenance value immediately. 2. **Add a partial index on `manager_domain`.** Item 2 of #4200 (queue- backed fan-out when a manager rotates their adagents.json) needs a manager → publishers reverse lookup. Building the partial index here is essentially free — the column is mostly NULL, so the index footprint is tiny — and means item 2 ships without another schema migration. 3. **Tighten `discovery_method` to required on the validate-publisher response schema.** `AdAgentsValidationResult.discovery_method` is already required and unconditionally set on every code path that reaches this response. Marking the OpenAPI schema `.optional()` was loose-on-the-server, leaking `T | undefined` into generated client types for a guarantee the server already meets. Now consumers get the tight contract. --- .../470_publisher_discovery_provenance.sql | 29 ++++++++++++++----- server/src/routes/registry-api.ts | 2 +- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/server/src/db/migrations/470_publisher_discovery_provenance.sql b/server/src/db/migrations/470_publisher_discovery_provenance.sql index 7a720518f0..845c417e49 100644 --- a/server/src/db/migrations/470_publisher_discovery_provenance.sql +++ b/server/src/db/migrations/470_publisher_discovery_provenance.sql @@ -4,12 +4,6 @@ -- and, when a managerdomain hop was used, which manager domain served the -- manifest. -- --- These columns are set by the crawler on every successful manifest cache --- write. Backfill is implicit: every publisher is re-crawled within the --- 60-minute cadence, so the new columns populate naturally on next fetch. --- Callers that see NULL can treat it as "not yet re-crawled since this --- migration" and fall back to existing adagents_valid signal. --- -- The CHECK constraint mirrors the DiscoveryMethod type in -- server/src/adagents-manager.ts so the DB rejects invalid values at -- write time rather than silently storing garbage. @@ -19,8 +13,29 @@ ALTER TABLE publishers CHECK (discovery_method IN ('direct', 'authoritative_location', 'ads_txt_managerdomain')), ADD COLUMN manager_domain TEXT; +-- Backfill: every successfully-validated publisher row that exists at the +-- time of this migration was necessarily discovered via the direct path — +-- the other two methods didn't exist before 4173/4204 landed. Stamping +-- 'direct' on those rows eliminates the otherwise-confusing NULL window +-- between this migration and the next crawl cycle, so /api/validate-publisher +-- and /api/registry/publisher return a stable provenance value immediately. +UPDATE publishers + SET discovery_method = 'direct' + WHERE discovery_method IS NULL + AND last_validated IS NOT NULL + AND adagents_valid IS TRUE; + +-- Partial index supports the manager → publishers reverse lookup planned +-- in #4200 item 2 (queue-backed fan-out when a manager rotates their +-- adagents.json). Building it here is essentially free — the column is +-- mostly NULL, so the index footprint is tiny — and means item 2 ships +-- without another schema migration. +CREATE INDEX idx_publishers_manager_domain + ON publishers (manager_domain) + WHERE manager_domain IS NOT NULL; + COMMENT ON COLUMN publishers.discovery_method IS - 'How the publisher''s adagents.json was discovered on the most recent successful crawl. ''direct'': publisher''s own /.well-known/ served the document. ''authoritative_location'': publisher''s stub redirected to a third-party canonical URL. ''ads_txt_managerdomain'': discovery fell back to a manager domain via ads.txt MANAGERDOMAIN delegation. NULL until first successful crawl after migration 470.'; + 'How the publisher''s adagents.json was discovered on the most recent successful crawl. ''direct'': publisher''s own /.well-known/ served the document. ''authoritative_location'': publisher''s stub redirected to a third-party canonical URL. ''ads_txt_managerdomain'': discovery fell back to a manager domain via ads.txt MANAGERDOMAIN delegation. Backfilled to ''direct'' for previously-validated rows.'; COMMENT ON COLUMN publishers.manager_domain IS 'The manager domain whose adagents.json was used to authorize this publisher''s agents. Non-NULL only when discovery_method = ''ads_txt_managerdomain''. Matches the MANAGERDOMAIN value from the publisher''s ads.txt. NULL for all other discovery methods.'; diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 2d94e566f7..b1eb9a7911 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -1045,7 +1045,7 @@ registry.registerPath({ valid: z.boolean(), domain: z.string(), url: z.string().optional(), - discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).optional().openapi({ + discovery_method: z.enum(["direct", "authoritative_location", "ads_txt_managerdomain"]).openapi({ description: "How the publisher's adagents.json was discovered. `ads_txt_managerdomain` indicates one-hop delegation via ads.txt MANAGERDOMAIN.", }), manager_domain: z.string().optional().openapi({ From 995895feec9d4b010d234fc86e8b96f193be5532 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 7 May 2026 17:51:46 -0400 Subject: [PATCH 4/4] fix(migration): backfill predicate uses source_type, not adagents_valid adagents_valid is an API-derived field, not a column on publishers. The right canonical signal for 'we have a successfully cached adagents.json for this row' is source_type = 'adagents_json' AND adagents_json IS NOT NULL. --- .../src/db/migrations/470_publisher_discovery_provenance.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/db/migrations/470_publisher_discovery_provenance.sql b/server/src/db/migrations/470_publisher_discovery_provenance.sql index 845c417e49..b6dbe62f21 100644 --- a/server/src/db/migrations/470_publisher_discovery_provenance.sql +++ b/server/src/db/migrations/470_publisher_discovery_provenance.sql @@ -22,8 +22,8 @@ ALTER TABLE publishers UPDATE publishers SET discovery_method = 'direct' WHERE discovery_method IS NULL - AND last_validated IS NOT NULL - AND adagents_valid IS TRUE; + AND source_type = 'adagents_json' + AND adagents_json IS NOT NULL; -- Partial index supports the manager → publishers reverse lookup planned -- in #4200 item 2 (queue-backed fan-out when a manager rotates their