fix(ingestion): R24e — drop ingestion_job_stages.job_id FK (resolve drift #8)

docs/database-drift-detection.md

@@ -123,8 +123,13 @@ live project need explicit operator approval.
    indexes after `pg_stat_user_indexes` scan verification; reshape 3
    (`import_batches_status_created_idx`, `ingestion_jobs_document_status_idx`,
    `ingestion_jobs_status_next_run_idx`).
-8. **Constraints**: add `ingestion_job_stages_job_id_fkey` to live; align the
-   `rag_visual_eval_*` document FK definitions.
+8. **Constraints**: ~~add `ingestion_job_stages_job_id_fkey` to live~~ RESOLVED
+   (audit R24e, migration `20260708140000`) — the FK was **removed from
+   schema.sql** instead of added to live. Live never had it; the edge agent
+   writes an `indexing_v3_agent_jobs` id into `job_id` (not an `ingestion_jobs`
+   id), and live carries 253 stage rows with no matching job, so a FK would
+   break the agent and fail VALIDATE. Still open: align the `rag_visual_eval_*`
+   document FK definitions.
 9. **`invoke_ingestion_worker`** hardcodes the project URL — migrate to the
    GUC pattern (`20260702160000` precedent).
 10. **Migration-chain fidelity** (affects Supabase Preview/branches, not

supabase/drift-allowlist.json

@@ -1,13 +1,6 @@
 {
   "_comment": "Known live-vs-schema.sql divergence measured 2026-07-07 (see docs/database-drift-detection.md). Every entry must carry a reason; check:drift reports matches as warnings and fails on anything NOT listed here. Remove entries as the reconciliation backlog lands.",
   "entries": [
-    {
-      "category": "constraints",
-      "key": "ingestion_job_stages.ingestion_job_stages_job_id_fkey",
-      "kind": "missing_live",
-      "reason": "FK declared in schema.sql and migrations but absent on live; add via approved migration (NOT VALID + VALIDATE pattern).",
-      "ref": "docs/database-drift-detection.md#reconciliation-backlog"
-    },
     {
       "category": "constraints",
       "key": "rag_visual_eval_cases.rag_visual_eval_cases_document_id_fkey",

supabase/drift-manifest.json

@@ -1,9 +1,9 @@
 {
-  "generated_at": "2026-07-08T06:45:01.039Z",
+  "generated_at": "2026-07-08T07:16:58.242Z",
   "generator": "scripts/generate-drift-manifest.ts",
   "postgres_image": "supabase/postgres:17.6.1.127",
-  "schema_sha256": "6174f3657e00b3a9ec11258cc599b956b6bdad805d7e37472710d0d34f4f1591",
-  "replay_seconds": 13,
+  "schema_sha256": "3cdee6b35b198f1e47db8e70f5e4f11c9828ad04f5c61a1e7184b4cbb7f43201",
+  "replay_seconds": 23,
   "snapshot": {
     "views": [
       {
@@ -6545,11 +6545,6 @@
         "name": "ingestion_job_stages_document_id_fkey",
         "table": "ingestion_job_stages"
       },
-      {
-        "def": "FOREIGN KEY (job_id) REFERENCES public.ingestion_jobs(id) ON DELETE CASCADE",
-        "name": "ingestion_job_stages_job_id_fkey",
-        "table": "ingestion_job_stages"
-      },
       {
         "def": "PRIMARY KEY (id)",
         "name": "ingestion_job_stages_pkey",

supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fkey.sql

@@ -0,0 +1,25 @@
+-- Audit R24e: drop ingestion_job_stages.job_id -> ingestion_jobs foreign key.
+--
+-- Resolves drift-reconciliation backlog item #8 in the direction the LIVE
+-- evidence supports (the opposite of the earlier "add the FK to live" plan):
+--
+--   * Live never had this FK; the constraint exists only in schema.sql and the
+--     migration chain, so fresh/preview databases diverge from live.
+--   * The edge agent's stageStart (supabase/functions/indexing-v3-agent) writes
+--     the indexing_v3_agent_jobs id into job_id, which can never satisfy a FK to
+--     ingestion_jobs — so on any database where the FK exists the agent's
+--     artifact-repair path dies at its first stage insert (FK 23503).
+--   * Live carries 253 stage rows whose job_id has no ingestion_jobs match
+--     (job deletions leave audit-log stages behind, with no FK to cascade them).
+--     Adding the FK NOT VALID + VALIDATE would fail VALIDATE against those rows.
+--
+-- job_id stays as an opaque correlation id (still NOT NULL). schema.sql is
+-- reconciled to omit the FK and supabase/drift-allowlist.json drops its
+-- now-obsolete entry in the same change. Idempotent: no-op on live (already
+-- absent), removes the constraint on branch/preview databases created from the
+-- pre-R24e migration chain.
+
+set search_path = public, extensions, pg_temp;
+
+alter table public.ingestion_job_stages
+  drop constraint if exists ingestion_job_stages_job_id_fkey;

supabase/schema.sql

@@ -473,7 +473,13 @@ create table if not exists public.ingestion_jobs (
 
 create table if not exists public.ingestion_job_stages (
   id uuid primary key default gen_random_uuid(),
-  job_id uuid not null references public.ingestion_jobs(id) on delete cascade,
+  -- Audit R24e: intentionally NOT a foreign key to ingestion_jobs. The edge
+  -- agent's stageStart writes the indexing_v3_agent_jobs id here (not an
+  -- ingestion_jobs id), and live already carries 253 stage rows whose job_id
+  -- has no ingestion_jobs match (job deletions leave audit-log stages behind).
+  -- A FK would fail VALIDATE against those rows and would break the agent on
+  -- fresh/preview databases. Kept as an opaque correlation id to match live.
+  job_id uuid not null,
   document_id uuid not null references public.documents(id) on delete cascade,
   stage_name text not null,
   stage_status text not null default 'started'

tests/supabase-schema.test.ts

@@ -285,10 +285,12 @@ describe("Supabase schema Data API grants", () => {
 
   it("keeps indexing-v3 enrichment claiming separate from raw ingestion jobs", () => {
     expect(schema).toContain("create table if not exists public.ingestion_job_stages");
-    expect(schema).toContain("job_id uuid not null references public.ingestion_jobs(id) on delete cascade");
-    expect(indexingV3AgentWorkerHardeningMigration).toContain(
-      "drop constraint if exists ingestion_job_stages_job_id_fkey",
-    );
+    // Audit R24e: job_id is intentionally NOT a foreign key to ingestion_jobs —
+    // the edge agent writes an indexing_v3_agent_jobs id there and live carries
+    // orphan stage rows, so a FK would break the agent and fail VALIDATE. The
+    // 20260625 hardening migration historically added the FK; 20260708140000
+    // drops it and schema.sql no longer declares it.
+    expect(schema).not.toContain("job_id uuid not null references public.ingestion_jobs(id)");
     expect(indexingV3AgentWorkerHardeningMigration).toContain(
       "add constraint ingestion_job_stages_job_id_fkey foreign key (job_id) references public.ingestion_jobs(id) on delete cascade",
     );