From 34740f6dce97ef743b1d1e16139af69e34cff178 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:19:45 +0800 Subject: [PATCH] =?UTF-8?q?fix(ingestion):=20R24e=20=E2=80=94=20drop=20ing?= =?UTF-8?q?estion=5Fjob=5Fstages.job=5Fid=20FK=20(resolve=20drift=20#8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live evidence, not the backlog's add-the-FK plan: live never had this FK, the edge agent writes an indexing_v3_agent_jobs id into job_id (never satisfies a FK to ingestion_jobs), and live carries 253 orphan stage rows that would fail VALIDATE. Removing the FK makes fresh/preview databases match live and unbreaks the agent's artifact-repair path there. - schema.sql: ingestion_job_stages.job_id is a plain not-null column (documented) - migration 20260708140000: drop constraint if exists (no-op on live; removes it on the branch/preview migration chain) - drift-allowlist: remove the now-obsolete FK entry (schema.sql matches live) - database-drift-detection.md: mark backlog item #8 resolved with rationale - supabase-schema.test: assert the FK is absent from schema.sql - drift manifest regenerated (clean scratch replay) Co-Authored-By: Claude Fable 5 --- docs/database-drift-detection.md | 9 +++++-- supabase/drift-allowlist.json | 7 ------ supabase/drift-manifest.json | 11 +++----- ..._drop_ingestion_job_stages_job_id_fkey.sql | 25 +++++++++++++++++++ supabase/schema.sql | 8 +++++- tests/supabase-schema.test.ts | 10 +++++--- 6 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fkey.sql diff --git a/docs/database-drift-detection.md b/docs/database-drift-detection.md index 956cd5567..99fbd5a1e 100644 --- a/docs/database-drift-detection.md +++ b/docs/database-drift-detection.md @@ -123,8 +123,13 @@ live project need explicit operator approval. indexes after `pg_stat_user_indexes` scan verification; reshape 3 (`import_batches_status_created_idx`, `ingestion_jobs_document_status_idx`, `ingestion_jobs_status_next_run_idx`). -8. **Constraints**: add `ingestion_job_stages_job_id_fkey` to live; align the - `rag_visual_eval_*` document FK definitions. +8. **Constraints**: ~~add `ingestion_job_stages_job_id_fkey` to live~~ RESOLVED + (audit R24e, migration `20260708140000`) — the FK was **removed from + schema.sql** instead of added to live. Live never had it; the edge agent + writes an `indexing_v3_agent_jobs` id into `job_id` (not an `ingestion_jobs` + id), and live carries 253 stage rows with no matching job, so a FK would + break the agent and fail VALIDATE. Still open: align the `rag_visual_eval_*` + document FK definitions. 9. **`invoke_ingestion_worker`** hardcodes the project URL — migrate to the GUC pattern (`20260702160000` precedent). 10. **Migration-chain fidelity** (affects Supabase Preview/branches, not diff --git a/supabase/drift-allowlist.json b/supabase/drift-allowlist.json index 6c888b048..12c020dde 100644 --- a/supabase/drift-allowlist.json +++ b/supabase/drift-allowlist.json @@ -1,13 +1,6 @@ { "_comment": "Known live-vs-schema.sql divergence measured 2026-07-07 (see docs/database-drift-detection.md). Every entry must carry a reason; check:drift reports matches as warnings and fails on anything NOT listed here. Remove entries as the reconciliation backlog lands.", "entries": [ - { - "category": "constraints", - "key": "ingestion_job_stages.ingestion_job_stages_job_id_fkey", - "kind": "missing_live", - "reason": "FK declared in schema.sql and migrations but absent on live; add via approved migration (NOT VALID + VALIDATE pattern).", - "ref": "docs/database-drift-detection.md#reconciliation-backlog" - }, { "category": "constraints", "key": "rag_visual_eval_cases.rag_visual_eval_cases_document_id_fkey", diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index 8c975b2bd..8761594a9 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-07-08T06:45:01.039Z", + "generated_at": "2026-07-08T07:16:58.242Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127", - "schema_sha256": "6174f3657e00b3a9ec11258cc599b956b6bdad805d7e37472710d0d34f4f1591", - "replay_seconds": 13, + "schema_sha256": "3cdee6b35b198f1e47db8e70f5e4f11c9828ad04f5c61a1e7184b4cbb7f43201", + "replay_seconds": 23, "snapshot": { "views": [ { @@ -6545,11 +6545,6 @@ "name": "ingestion_job_stages_document_id_fkey", "table": "ingestion_job_stages" }, - { - "def": "FOREIGN KEY (job_id) REFERENCES public.ingestion_jobs(id) ON DELETE CASCADE", - "name": "ingestion_job_stages_job_id_fkey", - "table": "ingestion_job_stages" - }, { "def": "PRIMARY KEY (id)", "name": "ingestion_job_stages_pkey", diff --git a/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fkey.sql b/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fkey.sql new file mode 100644 index 000000000..1db6888a2 --- /dev/null +++ b/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fkey.sql @@ -0,0 +1,25 @@ +-- Audit R24e: drop ingestion_job_stages.job_id -> ingestion_jobs foreign key. +-- +-- Resolves drift-reconciliation backlog item #8 in the direction the LIVE +-- evidence supports (the opposite of the earlier "add the FK to live" plan): +-- +-- * Live never had this FK; the constraint exists only in schema.sql and the +-- migration chain, so fresh/preview databases diverge from live. +-- * The edge agent's stageStart (supabase/functions/indexing-v3-agent) writes +-- the indexing_v3_agent_jobs id into job_id, which can never satisfy a FK to +-- ingestion_jobs — so on any database where the FK exists the agent's +-- artifact-repair path dies at its first stage insert (FK 23503). +-- * Live carries 253 stage rows whose job_id has no ingestion_jobs match +-- (job deletions leave audit-log stages behind, with no FK to cascade them). +-- Adding the FK NOT VALID + VALIDATE would fail VALIDATE against those rows. +-- +-- job_id stays as an opaque correlation id (still NOT NULL). schema.sql is +-- reconciled to omit the FK and supabase/drift-allowlist.json drops its +-- now-obsolete entry in the same change. Idempotent: no-op on live (already +-- absent), removes the constraint on branch/preview databases created from the +-- pre-R24e migration chain. + +set search_path = public, extensions, pg_temp; + +alter table public.ingestion_job_stages + drop constraint if exists ingestion_job_stages_job_id_fkey; diff --git a/supabase/schema.sql b/supabase/schema.sql index b5481e9d5..1a9dc65d9 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -473,7 +473,13 @@ create table if not exists public.ingestion_jobs ( create table if not exists public.ingestion_job_stages ( id uuid primary key default gen_random_uuid(), - job_id uuid not null references public.ingestion_jobs(id) on delete cascade, + -- Audit R24e: intentionally NOT a foreign key to ingestion_jobs. The edge + -- agent's stageStart writes the indexing_v3_agent_jobs id here (not an + -- ingestion_jobs id), and live already carries 253 stage rows whose job_id + -- has no ingestion_jobs match (job deletions leave audit-log stages behind). + -- A FK would fail VALIDATE against those rows and would break the agent on + -- fresh/preview databases. Kept as an opaque correlation id to match live. + job_id uuid not null, document_id uuid not null references public.documents(id) on delete cascade, stage_name text not null, stage_status text not null default 'started' diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 8e4abc7d1..28c5763c2 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -285,10 +285,12 @@ describe("Supabase schema Data API grants", () => { it("keeps indexing-v3 enrichment claiming separate from raw ingestion jobs", () => { expect(schema).toContain("create table if not exists public.ingestion_job_stages"); - expect(schema).toContain("job_id uuid not null references public.ingestion_jobs(id) on delete cascade"); - expect(indexingV3AgentWorkerHardeningMigration).toContain( - "drop constraint if exists ingestion_job_stages_job_id_fkey", - ); + // Audit R24e: job_id is intentionally NOT a foreign key to ingestion_jobs — + // the edge agent writes an indexing_v3_agent_jobs id there and live carries + // orphan stage rows, so a FK would break the agent and fail VALIDATE. The + // 20260625 hardening migration historically added the FK; 20260708140000 + // drops it and schema.sql no longer declares it. + expect(schema).not.toContain("job_id uuid not null references public.ingestion_jobs(id)"); expect(indexingV3AgentWorkerHardeningMigration).toContain( "add constraint ingestion_job_stages_job_id_fkey foreign key (job_id) references public.ingestion_jobs(id) on delete cascade", );