diff --git a/docs/database-drift-detection.md b/docs/database-drift-detection.md index c64fcf327..e8dd14f7b 100644 --- a/docs/database-drift-detection.md +++ b/docs/database-drift-detection.md @@ -108,7 +108,13 @@ live project need explicit operator approval. > operator-approved migration action. 0. **NEW — forward-codify the live-ahead retrieval RPCs** (was the "apply - 20260705210000" item, inverted). Live carries newer raw-SQL retrieval bodies + 20260705210000" item, inverted). **Staged runbook: + `docs/forward-codify-retrieval-rpcs-workorder.md`** — the read-only capture + query (`scripts/sql/capture-live-retrieval-rpcs.sql`), the fill-in-the-blanks + migration skeleton, the byte-faithful validation, and the allowlist cleanup; + a guard test (`tests/forward-codify-retrieval-targets.test.ts`) keeps the + capture query and this allowlist in lockstep. The capture + apply steps still + need a quiescent live DB and operator approval. Live carries newer raw-SQL retrieval bodies than the repo: `match_document_chunks` (hnsw.ef_search=100 plpgsql wrapper), `match_document_chunks_text` / `match_document_table_facts_text` (richer multi-strategy), `match_document_chunks_hybrid` (left-join quality_score), diff --git a/docs/forward-codify-retrieval-rpcs-workorder.md b/docs/forward-codify-retrieval-rpcs-workorder.md index 336e41492..686c04ad4 100644 --- a/docs/forward-codify-retrieval-rpcs-workorder.md +++ b/docs/forward-codify-retrieval-rpcs-workorder.md @@ -82,6 +82,33 @@ REPLACE` too so schema.sql declares them. These security-definer RPCs require pa 7. **Remove** the now-resolved `functions` entries from `supabase/drift-allowlist.json` (the `match_document_*` "live-ahead" set + the codified live-only functions) and regenerate the manifest. +## Committed capture artifact + rot guard + +The read-only capture in Step 1 is also committed as a runnable file so the +target set cannot silently drift from the machine-checked source of truth: + +- **`scripts/sql/capture-live-retrieval-rpcs.sql`** — `pg_get_functiondef` for + the exact `regprocedure` signatures of the retrieval functions currently + flagged `LIVE IS AHEAD` in `supabase/drift-allowlist.json` (7 as of + 2026-07-13, now including `repair_strict_enrichment_gate_batch`, which post-dates + the 2026-07-12 fingerprint table above). Pinned `search_path = ''` so the + signatures match the allowlist / `schema_drift_snapshot()` rendering exactly. + Re-run it at execution time per Step 1 and reconcile against the fingerprints. +- **`tests/forward-codify-retrieval-targets.test.ts`** — offline guard (runs in + `verify:cheap`) asserting that capture query's target set equals the allowlist + `LIVE IS AHEAD` retrieval entries, so reconciling a sibling (removing its + allowlist entry) forces a matching edit to the capture query and this file. + +The committed query covers **only** the `LIVE IS AHEAD` retrieval subset (its +guard invariant). It is **not** the full codification set: the live-only +(`unexpected_live`) and `(verify)` functions in the fingerprint table above — +`get_visual_evidence_cards`, `run_visual_eval_case`, `run_all_visual_eval_cases`, +`repair_enrichment_quality_batch`, `match_document_index_units_hybrid`, +`match_document_memory_cards_hybrid[_v2]` — are captured from that table per +Step 1, not from this query. The fingerprint table remains the point-in-time +reference snapshot; the committed query is the maintained, +always-in-sync-with-the-allowlist capture of its subset. + ## Not in this work-order (separate allowlist backlog) The allowlist also carries, with the same boilerplate reason, non-retrieval drift that should be triaged diff --git a/scripts/sql/capture-live-retrieval-rpcs.sql b/scripts/sql/capture-live-retrieval-rpcs.sql new file mode 100644 index 000000000..8476af969 --- /dev/null +++ b/scripts/sql/capture-live-retrieval-rpcs.sql @@ -0,0 +1,67 @@ +-- capture-live-retrieval-rpcs.sql (READ-ONLY) +-- +-- Captures the VERBATIM live bodies of the retrieval RPCs that are still flagged +-- "LIVE IS AHEAD" in supabase/drift-allowlist.json, so their definitions can be +-- forward-codified into a migration + supabase/schema.sql WITHOUT hand-authoring +-- them. These bodies are complex, have diverged in both directions (e.g. the +-- fail-closed retrieval_owner_matches predicate was applied to live while live +-- also grew richer multi-strategy candidate sets), and are under active +-- concurrent multi-session editing — so only a verbatim capture is correct. A +-- hand-written body is exactly what got migration 20260705210000 neutralized. +-- +-- SCOPE: this covers ONLY the allowlist "LIVE IS AHEAD" retrieval subset — the +-- guard-tested set (tests/forward-codify-retrieval-targets.test.ts). It is NOT +-- the complete codification set. The runbook also codifies live-only +-- (unexpected_live) and "verify" functions — get_visual_evidence_cards, +-- run_visual_eval_case, run_all_visual_eval_cases, repair_enrichment_quality_batch, +-- match_document_index_units_hybrid, match_document_memory_cards_hybrid[_v2]. +-- Capture those from the fingerprint table in +-- docs/forward-codify-retrieval-rpcs-workorder.md, not here. +-- +-- Full procedure: docs/forward-codify-retrieval-rpcs-workorder.md +-- Backlog item 0: docs/database-drift-detection.md#reconciliation-backlog +-- +-- HOW TO RUN — this is a provider action (it reads the live project); get +-- explicit approval first and run it only when the live DB is QUIESCENT (no +-- concurrent edits to these functions). It changes nothing — it only reads +-- pg_proc. Run it via the Supabase Dashboard SQL editor, an approved +-- service-role SQL path, or the Supabase MCP execute_sql tool. +-- +-- Expect EXACTLY 7 rows back (one per target). A target that no longer resolves +-- on live (renamed, dropped, or already reconciled) is filtered out and simply +-- yields FEWER rows — the query does not error. If you get fewer than 7, find +-- which with the "missing signatures" snippet in the work-order and reconcile +-- the target set + allowlist before continuing. +-- +-- search_path is pinned to '' so oid::regprocedure renders fully-qualified and +-- matches the allowlist keys / schema_drift_snapshot() signatures exactly, and +-- so pg_get_functiondef emits fully-qualified, replay-safe text (the same style +-- as migrations 20260701140631 / 20260707000000). This is cosmetic, not +-- correctness-critical: check:drift hashes pg_get_functiondef under search_path +-- '' on both live and the schema.sql replay, so a verbatim body always hashes +-- equal regardless of how it was rendered at capture time. + +set search_path = ''; + +select + r.signature, + pg_get_functiondef(r.proc) as definition +from ( + select t.signature, to_regprocedure(t.signature) as proc + from ( + -- >>> forward-codify targets >>> (kept in lockstep with the "LIVE IS AHEAD" + -- retrieval entries in supabase/drift-allowlist.json — see + -- tests/forward-codify-retrieval-targets.test.ts) + values + ('public.get_related_document_metadata(uuid[],uuid)'), + ('public.match_document_chunks(extensions.vector,integer,double precision,uuid,uuid)'), + ('public.match_document_chunks_hybrid(extensions.vector,text,integer,double precision,uuid[],uuid)'), + ('public.match_document_chunks_text(text,integer,uuid[],uuid)'), + ('public.match_document_table_facts_text(text,integer,uuid[],uuid)'), + ('public.match_documents_for_query(text,integer,uuid)'), + ('public.repair_strict_enrichment_gate_batch(integer)') + -- <<< forward-codify targets <<< + ) as t(signature) +) as r +where r.proc is not null +order by r.signature; diff --git a/tests/forward-codify-retrieval-targets.test.ts b/tests/forward-codify-retrieval-targets.test.ts new file mode 100644 index 000000000..2f7b41503 --- /dev/null +++ b/tests/forward-codify-retrieval-targets.test.ts @@ -0,0 +1,54 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +// Guards drift backlog item 0 (docs/forward-codify-retrieval-rpcs-workorder.md): +// the read-only capture query scripts/sql/capture-live-retrieval-rpcs.sql must +// target exactly the retrieval RPCs still flagged "LIVE IS AHEAD" in the drift +// allowlist. Reconciling a sibling (removing its allowlist entry) then forces a +// matching edit to the capture query, so the work-order artifact cannot silently +// rot as the backlog is worked down. + +const root = join(__dirname, ".."); +const read = (relative: string) => readFileSync(join(root, relative), "utf8"); + +// Allowlist entries the work-order treats as its authoritative target set. +const LIVE_AHEAD_PREFIX = "LIVE IS AHEAD"; + +type AllowlistEntry = { category: string; kind: string; key: string; reason: string }; + +function liveAheadRetrievalSignatures(): string[] { + const { entries } = JSON.parse(read("supabase/drift-allowlist.json")) as { entries: AllowlistEntry[] }; + return entries + .filter((e) => e.category === "functions" && e.kind === "mismatch" && e.reason.startsWith(LIVE_AHEAD_PREFIX)) + .map((e) => e.key) + .sort(); +} + +function captureQuerySignatures(): string[] { + const sql = read("scripts/sql/capture-live-retrieval-rpcs.sql"); + const start = sql.indexOf(">>> forward-codify targets >>>"); + const end = sql.indexOf("<<< forward-codify targets <<<"); + expect(start, "capture query is missing its `>>> forward-codify targets >>>` marker").toBeGreaterThanOrEqual(0); + expect(end, "capture query is missing its `<<< forward-codify targets <<<` marker").toBeGreaterThan(start); + const region = sql.slice(start, end); + // Signatures are single-quoted VALUES literals; regprocedure text never + // contains a single quote, so this is an exact extraction. + const matches = [...region.matchAll(/'([^']+)'/g)].map((m) => m[1]); + return matches.sort(); +} + +describe("forward-codify retrieval target set stays in sync with the drift allowlist", () => { + it("finds a non-empty set of live-ahead retrieval functions in the allowlist", () => { + expect(liveAheadRetrievalSignatures().length).toBeGreaterThan(0); + }); + + it("the capture query lists every live-ahead retrieval signature, and only those", () => { + expect(captureQuerySignatures()).toEqual(liveAheadRetrievalSignatures()); + }); + + it("the capture query has no duplicate target signatures", () => { + const signatures = captureQuerySignatures(); + expect(new Set(signatures).size).toBe(signatures.length); + }); +});