diff --git a/docs/database-drift-detection.md b/docs/database-drift-detection.md index 956cd5567..c09d542d6 100644 --- a/docs/database-drift-detection.md +++ b/docs/database-drift-detection.md @@ -123,8 +123,14 @@ live project need explicit operator approval. indexes after `pg_stat_user_indexes` scan verification; reshape 3 (`import_batches_status_created_idx`, `ingestion_jobs_document_status_idx`, `ingestion_jobs_status_next_run_idx`). -8. **Constraints**: add `ingestion_job_stages_job_id_fkey` to live; align the - `rag_visual_eval_*` document FK definitions. +8. **Constraints**: ~~add `ingestion_job_stages_job_id_fkey` to live~~ — + **reversed (R24e, 2026-07-08):** the FK was **removed from `schema.sql`** + (migration `20260708140000`) instead of added to live. Live has ~253 orphan + stage rows and 0 rows whose `job_id` resolves to an `ingestion_jobs` row, and + the column holds `indexing_v3_agent_jobs` ids — so adding + VALIDATE-ing the + FK would destroy stage-log history and break the edge agent (see + `docs/ingestion-state-machine.md` R24e). Allowlist entry removed. Still open: + align the `rag_visual_eval_*` document FK definitions. 9. **`invoke_ingestion_worker`** hardcodes the project URL — migrate to the GUC pattern (`20260702160000` precedent). 10. **Migration-chain fidelity** (affects Supabase Preview/branches, not diff --git a/supabase/drift-allowlist.json b/supabase/drift-allowlist.json index 6c888b048..12c020dde 100644 --- a/supabase/drift-allowlist.json +++ b/supabase/drift-allowlist.json @@ -1,13 +1,6 @@ { "_comment": "Known live-vs-schema.sql divergence measured 2026-07-07 (see docs/database-drift-detection.md). Every entry must carry a reason; check:drift reports matches as warnings and fails on anything NOT listed here. Remove entries as the reconciliation backlog lands.", "entries": [ - { - "category": "constraints", - "key": "ingestion_job_stages.ingestion_job_stages_job_id_fkey", - "kind": "missing_live", - "reason": "FK declared in schema.sql and migrations but absent on live; add via approved migration (NOT VALID + VALIDATE pattern).", - "ref": "docs/database-drift-detection.md#reconciliation-backlog" - }, { "category": "constraints", "key": "rag_visual_eval_cases.rag_visual_eval_cases_document_id_fkey", diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index 8c975b2bd..720134ce2 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -2,7 +2,7 @@ "generated_at": "2026-07-08T06:45:01.039Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127", - "schema_sha256": "6174f3657e00b3a9ec11258cc599b956b6bdad805d7e37472710d0d34f4f1591", + "schema_sha256": "430046a9221f4060286c2208800ef794eef407af8997fd4582f414de95e88c44", "replay_seconds": 13, "snapshot": { "views": [ @@ -6545,11 +6545,6 @@ "name": "ingestion_job_stages_document_id_fkey", "table": "ingestion_job_stages" }, - { - "def": "FOREIGN KEY (job_id) REFERENCES public.ingestion_jobs(id) ON DELETE CASCADE", - "name": "ingestion_job_stages_job_id_fkey", - "table": "ingestion_job_stages" - }, { "def": "PRIMARY KEY (id)", "name": "ingestion_job_stages_pkey", diff --git a/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fk.sql b/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fk.sql new file mode 100644 index 000000000..42db4a321 --- /dev/null +++ b/supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fk.sql @@ -0,0 +1,25 @@ +-- R24e: remove the phantom ingestion_job_stages.job_id -> ingestion_jobs FK. +-- +-- Rationale (docs/ingestion-state-machine.md finding R24e; +-- docs/ingestion-concurrency-fix-workorder.md): schema.sql and migration +-- 20260625000000 declare +-- job_id uuid not null references public.ingestion_jobs(id) on delete cascade, +-- but this constraint is ABSENT on the live project, and the column holds +-- indexing_v3_agent_jobs ids (the edge agent's stageStart writes the agent-job +-- id into job_id), not ingestion_jobs ids. On any schema.sql- or +-- migration-provisioned environment the FK makes every needs-work agent run die +-- at its first stage insert (FK violation 23503) and burn attempts to terminal +-- failed. Live carries ~253 orphan stage rows and 0 rows whose job_id resolves +-- to an ingestion_jobs row, so ADDING + VALIDATE-ing the FK (the prior +-- drift-allowlist plan) would both destroy stage-log history and break the +-- agent. Drop it so fresh/preview environments match live. Document cleanup is +-- still covered by ingestion_job_stages.document_id -> documents ON DELETE +-- CASCADE, which is the FK live actually has. +-- +-- Idempotent: a no-op on live (already absent). schema.sql is reconciled to +-- match this and supabase/drift-manifest.json is regenerated in the same change; +-- the matching supabase/drift-allowlist.json entry is removed. Supersedes +-- reconciliation-backlog item #8 in docs/database-drift-detection.md. + +alter table if exists public.ingestion_job_stages + drop constraint if exists ingestion_job_stages_job_id_fkey; diff --git a/supabase/schema.sql b/supabase/schema.sql index b5481e9d5..484c41b84 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -473,7 +473,10 @@ create table if not exists public.ingestion_jobs ( create table if not exists public.ingestion_job_stages ( id uuid primary key default gen_random_uuid(), - job_id uuid not null references public.ingestion_jobs(id) on delete cascade, + -- R24e: no FK to ingestion_jobs. Live has no such constraint and job_id holds + -- indexing_v3_agent_jobs ids, not ingestion_jobs ids (see migration + -- 20260708140000). document_id -> documents ON DELETE CASCADE is the real FK. + job_id uuid not null, document_id uuid not null references public.documents(id) on delete cascade, stage_name text not null, stage_status text not null default 'started' diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 8e4abc7d1..bbd1b5c2c 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -26,6 +26,10 @@ const indexingV3AgentWorkerHardeningMigration = readFileSync( new URL("../supabase/migrations/20260625000000_indexing_v3_agent_worker_hardening.sql", import.meta.url), "utf8", ).replace(/\s+/g, " "); +const dropStageJobIdFkMigration = readFileSync( + new URL("../supabase/migrations/20260708140000_drop_ingestion_job_stages_job_id_fk.sql", import.meta.url), + "utf8", +).replace(/\s+/g, " "); const atomicStrictCompletionMigration = readFileSync( new URL("../supabase/migrations/20260625033944_atomic_strict_enrichment_completion.sql", import.meta.url), "utf8", @@ -285,13 +289,17 @@ describe("Supabase schema Data API grants", () => { it("keeps indexing-v3 enrichment claiming separate from raw ingestion jobs", () => { expect(schema).toContain("create table if not exists public.ingestion_job_stages"); - expect(schema).toContain("job_id uuid not null references public.ingestion_jobs(id) on delete cascade"); - expect(indexingV3AgentWorkerHardeningMigration).toContain( - "drop constraint if exists ingestion_job_stages_job_id_fkey", - ); + // R24e: schema.sql no longer declares a job_id -> ingestion_jobs FK. Live has + // none, and job_id holds indexing_v3_agent_jobs ids, not ingestion_jobs ids, + // so the FK would break the edge agent. The historical migration + // 20260625000000 added it; 20260708140000 drops it so fresh/preview + // environments match live. + expect(schema).toContain("job_id uuid not null,"); + expect(schema).not.toContain("job_id uuid not null references public.ingestion_jobs(id) on delete cascade"); expect(indexingV3AgentWorkerHardeningMigration).toContain( "add constraint ingestion_job_stages_job_id_fkey foreign key (job_id) references public.ingestion_jobs(id) on delete cascade", ); + expect(dropStageJobIdFkMigration).toContain("drop constraint if exists ingestion_job_stages_job_id_fkey"); expect(schema).toContain("drop index if exists public.ingestion_job_stages_doc_idx"); expect(schema).toContain("create index if not exists ingestion_job_stages_document_started_idx"); for (const sql of [schema, indexingV3AgentJobsMigration]) {