diff --git a/.env.example b/.env.example index 62cee8871..273e16f09 100644 --- a/.env.example +++ b/.env.example @@ -6,9 +6,14 @@ SUPABASE_PROJECT_REF=sjrfecxgysukkwxsowpy SUPABASE_PROJECT_NAME=Clinical KB Database NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=your-publishable-or-anon-key SUPABASE_SERVICE_ROLE_KEY=your-service-role-key +# Direct Postgres connection string for scripts/migrations that need SQL access. +# Server-only; never expose in client code or NEXT_PUBLIC_ vars. +SUPABASE_DB_URL=postgresql://postgres:password@db.sjrfecxgysukkwxsowpy.supabase.co:5432/postgres # Edge Function only (indexing-v3-agent cron auth). Not read by Next.js env.ts. # Set in Supabase Edge Function secrets / deployment env, not in the browser bundle. INDEXING_V3_AGENT_SECRET=your-long-random-cron-shared-secret +# Secret required for /api/health?deep=1 Supabase probe (x-health-deep-token header). +HEALTH_DEEP_PROBE_SECRET=your-long-random-health-deep-probe-secret # Local-only no-auth mode (development only). # Enable one or both of these to load real data without per-request browser auth: diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 7407a6c4b..f6a53a4ee 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -95,6 +95,15 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went - Auth server is capped at 10 absolute DB connections (Supabase advisor); switch to percentage-based allocation in the dashboard before scaling instance size (not settable via SQL/MCP). - `storage_cleanup_jobs` live indexes drifted from `supabase/schema.sql`: live carries legacy auto-names (`storage_cleanup_jobs_document_id_idx`, `storage_cleanup_jobs_owner_id_idx`, a non-partial `storage_cleanup_jobs_status_created_idx`) that the hardening defs superseded. Migration `20260703030000_reconcile_storage_cleanup_jobs_indexes` is **prepared but NOT applied** — it drops the legacy names and (re)creates the intended named/partial indexes to match schema.sql. Functional-not-broken (the document_id FK is covered), so apply to live only with explicit approval. `20260703000000`/`010000` are also absent from live `schema_migrations` and will self-heal on the next `supabase db push`. +## Live database drift reconciliation (2026-07-05) + +- **RESOLVED:** `indexing_v3_agent_jobs` table + `claim_indexing_v3_agent_jobs` + `update_indexing_v3_agent_job_status` were recorded as applied but absent on live. Migration `20260705230000_reconcile_live_database_drift` idempotently re-applied them; live verified post-push. +- **RESOLVED:** `match_document_embedding_fields_text` codified with service_role-only execute. +- **RESOLVED:** `rag_visual_eval_*` tables codified with service_role-only RLS. +- **RESOLVED:** Live-only `20260705133000_tighten_search_document_chunks_owner_scope` mirrored in `schema.sql`. +- **Edge function follow-up:** deploy `indexing-v3-agent` after merge so JSONB status RPC parsing is live. +- **Operator-only:** publishable key rotation (`docs/archive/operator-decisions-2026-07-04.md`). + ## PR merge gate: tiered CI + required checks (2026-07-02) - CI is now two parallel PR jobs instead of one serial 6-7 minute job: `verify` (runtime alignment, edge typecheck, CI-safe production readiness, lint, typecheck, unit tests with coverage gate, build — ~3 min) and `ui-smoke` (Chromium Playwright smoke against its own dev server — ~4.5 min). Wall-clock PR feedback drops to the slower of the two, and a flaky smoke rerun no longer repeats lint/typecheck/tests/build. diff --git a/docs/supabase-migration-reconciliation.md b/docs/supabase-migration-reconciliation.md index 37ba5dbd3..e6c51c7fa 100644 --- a/docs/supabase-migration-reconciliation.md +++ b/docs/supabase-migration-reconciliation.md @@ -1,6 +1,6 @@ # Supabase Migration Reconciliation -Last reviewed: 2026-07-04 +Last reviewed: 2026-07-05 Target project: Clinical KB Database (`sjrfecxgysukkwxsowpy`) @@ -27,7 +27,15 @@ These previously local-only versions were verified in the live project history b ## Current Status (July 2026) -The repo now includes additional July 2026 migrations beyond the June checkpoint above, including: +Migration `20260705230000_reconcile_live_database_drift.sql` codifies live-only drift discovered 2026-07-05: + +- `indexing_v3_agent_jobs` table and claim/update RPCs (recorded as applied in history but absent on live at inspection time) +- `match_document_embedding_fields_text` RPC with service-role-only execute grants (was present on live with anon/auth execute) +- `rag_visual_eval_cases` / `rag_visual_eval_runs` tables with service-role-only RLS (were present on live without RLS) + +`supabase/schema.sql` has been reconciled to match. Apply the migration through the normal linked workflow when ready; do not use raw dashboard SQL for retrieval RPCs. + +The repo also includes additional July 2026 migrations beyond the June checkpoint above, including: - Retrieval RPC codification and hybrid execution smoke (`20260701140631`, related July 1 fixes) - Legacy vector index drops and `search_schema_health()` reconciliation (`20260702014803`, `20260702021604`) diff --git a/package.json b/package.json index 6bb92b079..76bbc5f6c 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,6 @@ "pdfjs-dist": "^6.1.200", "pdfkit": "^0.19.1", "postcss": "^8.5.15", - "postgres": "3.4.9", "react": "19.2.7", "react-dom": "19.2.7", "zod": "^4.4.3" diff --git a/scripts/check-document-label-governance.ts b/scripts/check-document-label-governance.ts index c418edf4c..a6615f06f 100644 --- a/scripts/check-document-label-governance.ts +++ b/scripts/check-document-label-governance.ts @@ -175,7 +175,7 @@ async function main() { console.log(`Low confidence generated labels: ${report.analytics.lowConfidence}`); console.log(`Quality warnings: ${report.analytics.qualityIssues.length}`); console.log(`Blocking quality issues: ${report.analytics.blockingQualityIssues.length}`); - console.log(`Missing gold-label rows: ${report.analytics.missingGoldLabels.length}`); + console.log(`Missing gold-label rows (advisory): ${report.analytics.missingGoldLabels.length}`); console.log( `Relevance checks: ${report.relevanceChecks.filter((check) => check.passed).length}/${report.relevanceChecks.length} passed`, ); diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index f70d0264b..0a0ab8c3a 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -1,24 +1,34 @@ +import { timingSafeEqual } from "node:crypto"; import { NextResponse } from "next/server"; import { env, isDemoMode } from "@/lib/env"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -// Liveness/readiness probe for load balancers and uptime monitors. It reports -// whether the server is configured to operate for real (Supabase + OpenAI present) -// and, with ?deep=1, whether Supabase is actually reachable. The payload exposes only -// boolean configuration presence and operational status — never secret values. +function allowDeepHealthProbe(request: Request): boolean { + const secret = env.HEALTH_DEEP_PROBE_SECRET; + if (!secret) return false; + const token = request.headers.get("x-health-deep-token"); + if (!token) return false; + if (token.length !== secret.length) return false; + const expected = Buffer.from(secret, "utf8"); + const received = Buffer.from(token, "utf8"); + return timingSafeEqual(expected, received); +} + export async function GET(request: Request) { const deep = new URL(request.url).searchParams.get("deep") === "1"; const supabaseConfigured = Boolean(env.NEXT_PUBLIC_SUPABASE_URL && env.SUPABASE_SERVICE_ROLE_KEY); - const checks: Record = { + const checks: Record = { supabaseConfig: supabaseConfigured ? "ok" : "missing", openaiConfig: env.OPENAI_API_KEY ? "ok" : "missing", }; if (deep) { - if (supabaseConfigured && !isDemoMode()) { + if (!allowDeepHealthProbe(request)) { + checks.supabase = "unauthorized"; + } else if (supabaseConfigured && !isDemoMode()) { try { const [{ createAdminClient }, { probeSupabaseHealth }] = await Promise.all([ import("@/lib/supabase/admin"), @@ -34,7 +44,9 @@ export async function GET(request: Request) { } } - const ready = !Object.values(checks).includes("missing") && !Object.values(checks).includes("error"); + const ready = !Object.values(checks).some( + (value) => value === "missing" || value === "error" || value === "unauthorized", + ); return NextResponse.json( { diff --git a/src/lib/document-label-governance.ts b/src/lib/document-label-governance.ts index 3b9bb4288..ea1e895e8 100644 --- a/src/lib/document-label-governance.ts +++ b/src/lib/document-label-governance.ts @@ -381,9 +381,6 @@ export function buildDocumentLabelGovernanceReport(documents: LabelGovernanceDoc analytics, qaSample, relevanceChecks, - passed: - analytics.blockingQualityIssues.length === 0 && - analytics.missingGoldLabels.length === 0 && - relevanceChecks.every((check) => check.passed), + passed: analytics.blockingQualityIssues.length === 0 && relevanceChecks.every((check) => check.passed), }; } diff --git a/src/lib/env.ts b/src/lib/env.ts index f6acba54f..b17748051 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -7,6 +7,8 @@ const envSchema = z.object({ SUPABASE_PROJECT_REF: z.string().optional(), SUPABASE_PROJECT_NAME: z.string().optional(), SUPABASE_SERVICE_ROLE_KEY: z.string().optional(), + SUPABASE_DB_URL: z.string().url().optional(), + HEALTH_DEEP_PROBE_SECRET: z.string().min(16).optional(), NEXT_PUBLIC_LOCAL_NO_AUTH: z.enum(["true", "false"]).optional().default("false"), LOCAL_NO_AUTH: z.enum(["true", "false"]).optional().default("false"), LOCAL_NO_AUTH_OWNER_EMAIL: z.string().optional(), diff --git a/supabase/functions/indexing-v3-agent/index.ts b/supabase/functions/indexing-v3-agent/index.ts index aa66e9354..c55ba7caa 100644 --- a/supabase/functions/indexing-v3-agent/index.ts +++ b/supabase/functions/indexing-v3-agent/index.ts @@ -1882,7 +1882,7 @@ async function updateAgentJobStatus( error: string | null = null, nextRunAt: string | null = null, ): Promise { - const rows = await sql>` + const rows = await sql` select * from public.update_indexing_v3_agent_job_status( ${job.document_id}::uuid, @@ -1891,7 +1891,8 @@ async function updateAgentJobStatus( ${nextRunAt}::timestamptz ) `; - if (!rows[0]?.ok) { + const result = parseJobStatusRpcResult(rows[0], "update_indexing_v3_agent_job_status"); + if (!result?.ok) { throw new Error(`Failed to update indexing_v3_agent_jobs status to ${status} for document ${job.document_id}`); } } diff --git a/supabase/migrations/20260705133000_tighten_search_document_chunks_owner_scope.sql b/supabase/migrations/20260705133000_tighten_search_document_chunks_owner_scope.sql index 473fdb90c..024203b98 100644 --- a/supabase/migrations/20260705133000_tighten_search_document_chunks_owner_scope.sql +++ b/supabase/migrations/20260705133000_tighten_search_document_chunks_owner_scope.sql @@ -1,6 +1,6 @@ --- Tighten search_document_chunks owner scoping so null p_owner_id only matches public --- documents (owner_id IS NULL) and authenticated callers can search both owned and public --- documents without matching other owners' private rows. +-- Codify live migration tighten_search_document_chunks_owner_scope (20260705133000). +-- Fail closed: null p_owner_id may only search public documents (owner_id is null). +-- Authenticated callers can search both owned and public documents without matching other owners' private rows. create or replace function public.search_document_chunks( p_document_id uuid, @@ -69,4 +69,5 @@ as $$ limit least(greatest(match_count, 1), 80); $$; +revoke execute on function public.search_document_chunks(uuid, text, integer, uuid) from public, anon, authenticated; grant execute on function public.search_document_chunks(uuid, text, integer, uuid) to service_role; diff --git a/supabase/migrations/20260705230000_reconcile_live_database_drift.sql b/supabase/migrations/20260705230000_reconcile_live_database_drift.sql new file mode 100644 index 000000000..b5c3202b6 --- /dev/null +++ b/supabase/migrations/20260705230000_reconcile_live_database_drift.sql @@ -0,0 +1,264 @@ +-- Reconcile live database drift discovered 2026-07-05: +-- 1. indexing_v3_agent_jobs table + claim/update RPCs recorded as applied but absent on live +-- 2. match_document_embedding_fields_text exists on live with anon/auth execute (codify + lock down) +-- 3. rag_visual_eval_* tables exist on live without RLS (codify + enable service_role-only RLS) + +set search_path = public, extensions, pg_catalog; + +create table if not exists public.indexing_v3_agent_jobs ( + id uuid primary key default gen_random_uuid(), + document_id uuid not null references public.documents(id) on delete cascade, + status text not null default 'pending' + check (status in ('pending', 'processing', 'completed', 'failed', 'needs_enrichment_artifacts')), + enrichment_status text not null default 'pending' + check (enrichment_status in ('pending', 'processing', 'completed', 'failed', 'needs_enrichment_artifacts')), + attempt_count integer not null default 0, + max_attempts integer not null default 3, + locked_by text, + locked_at timestamptz, + next_run_at timestamptz, + version text not null default 'visual-core-v3', + last_error text, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create unique index if not exists indexing_v3_agent_jobs_document_id_idx + on public.indexing_v3_agent_jobs(document_id); + +create index if not exists indexing_v3_agent_jobs_claim_idx + on public.indexing_v3_agent_jobs(status, enrichment_status, next_run_at, id) + where status not in ('completed', 'needs_enrichment_artifacts'); + +create index if not exists indexing_v3_agent_jobs_locked_at_idx + on public.indexing_v3_agent_jobs(locked_at) + where status = 'processing'; + +alter table public.indexing_v3_agent_jobs enable row level security; + +drop policy if exists "indexing v3 agent jobs service role all" on public.indexing_v3_agent_jobs; +create policy "indexing v3 agent jobs service role all" + on public.indexing_v3_agent_jobs + for all to service_role + using (true) + with check (true); + +grant select, insert, update, delete + on table public.indexing_v3_agent_jobs to service_role; + +insert into public.indexing_v3_agent_jobs ( + document_id, status, enrichment_status, attempt_count, max_attempts, + locked_by, locked_at, next_run_at, version, last_error, metadata, created_at, updated_at +) +select + d.id, + case + when coalesce(d.metadata->>'indexing_v3_agent_status', 'pending') in + ('completed', 'needs_enrichment_artifacts', 'failed') + then coalesce(d.metadata->>'indexing_v3_agent_status', 'pending') + when coalesce(d.metadata->>'indexing_v3_agent_status', 'pending') in ('deferred', 'retry_pending') + then 'pending' + when coalesce(d.metadata->>'indexing_v3_agent_status', '') = 'processing' + and ( + nullif(d.metadata->>'indexing_v3_agent_locked_at', '') is null + or (d.metadata->>'indexing_v3_agent_locked_at')::timestamptz < now() - interval '2 hours' + ) + then 'pending' + else 'pending' + end, + coalesce(d.metadata->>'enrichment_status', 'pending'), + case when coalesce(d.metadata->>'indexing_v3_agent_attempt_count', '') ~ '^[0-9]+$' + then (d.metadata->>'indexing_v3_agent_attempt_count')::integer else 0 end, + greatest(case when coalesce(d.metadata->>'indexing_v3_agent_max_attempts', '') ~ '^[0-9]+$' + then (d.metadata->>'indexing_v3_agent_max_attempts')::integer else 3 end, 1), + nullif(d.metadata->>'indexing_v3_agent_locked_by', ''), + case when coalesce(d.metadata->>'indexing_v3_agent_locked_at', '') ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + then (d.metadata->>'indexing_v3_agent_locked_at')::timestamptz else null end, + case when coalesce(d.metadata->>'indexing_v3_agent_next_run_at', '') ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + then (d.metadata->>'indexing_v3_agent_next_run_at')::timestamptz else null end, + coalesce(nullif(d.metadata->>'indexing_v3_agent_version', ''), 'visual-core-v3'), + nullif(d.metadata->>'indexing_v3_agent_last_error', ''), + '{}'::jsonb, + coalesce(d.created_at, now()), + coalesce(d.updated_at, now()) +from public.documents d +where d.metadata ? 'indexing_v3_agent_status' +on conflict (document_id) do nothing; + +create or replace function public.claim_indexing_v3_agent_jobs( + p_worker_id text, + p_claim_limit integer default 1, + p_stale_after_minutes integer default 45 +) +returns table ( + id uuid, document_id uuid, batch_id uuid, status text, stage text, progress integer, + error_message text, attempt_count integer, max_attempts integer, locked_at timestamptz, + locked_by text, documents jsonb +) +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +begin + insert into public.indexing_v3_agent_jobs ( + document_id, status, enrichment_status, next_run_at, version, metadata, created_at, updated_at + ) + select d.id, 'pending', coalesce(d.metadata->>'enrichment_status', 'pending'), + case when coalesce(d.metadata->>'indexing_v3_agent_next_run_at', '') ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + then (d.metadata->>'indexing_v3_agent_next_run_at')::timestamptz else null end, + coalesce(nullif(d.metadata->>'indexing_v3_agent_version', ''), 'visual-core-v3'), + '{}'::jsonb, coalesce(d.created_at, now()), now() + from public.documents d + where d.status = 'indexed' + and d.metadata ? 'indexing_v3_agent_status' + and coalesce(d.metadata->>'indexing_v3_agent_status', 'pending') + not in ('completed', 'needs_enrichment_artifacts') + on conflict (document_id) do nothing; + + return query + with eligible_jobs as ( + select j.id, j.document_id, j.attempt_count, j.max_attempts + from public.indexing_v3_agent_jobs j + join public.documents d on d.id = j.document_id and d.status = 'indexed' + where j.status not in ('completed', 'needs_enrichment_artifacts') + and j.enrichment_status in ('pending', 'failed', 'processing') + and j.attempt_count < j.max_attempts + and coalesce(j.next_run_at, now()) <= now() + and (j.status <> 'processing' or j.locked_at is null + or j.locked_at < now() - make_interval(mins => p_stale_after_minutes)) + order by coalesce(j.next_run_at, j.updated_at), j.id + limit greatest(p_claim_limit, 1) + for update of j skip locked + ), + claimed_jobs as ( + update public.indexing_v3_agent_jobs j + set status = 'processing', enrichment_status = 'processing', locked_by = p_worker_id, + locked_at = now(), attempt_count = e.attempt_count + 1, last_error = null, + next_run_at = null, updated_at = now() + from eligible_jobs e where j.id = e.id returning j.* + ), + patched_documents as ( + update public.documents d + set metadata = jsonb_strip_nulls( + (coalesce(d.metadata, '{}'::jsonb) - 'indexing_v3_agent_next_run_at' - 'indexing_v3_agent_last_error') + || jsonb_build_object( + 'indexing_v3_agent_status', 'processing', + 'indexing_v3_agent_version', cj.version, + 'indexing_v3_agent_locked_by', p_worker_id, + 'indexing_v3_agent_locked_at', cj.locked_at, + 'indexing_v3_agent_attempt_count', cj.attempt_count, + 'indexing_v3_agent_max_attempts', cj.max_attempts, + 'indexing_v3_agent_updated_at', now(), + 'enrichment_status', 'processing' + ) + ), updated_at = now() + from claimed_jobs cj + where d.id = cj.document_id and d.status = 'indexed' + returning d.*, cj.id as job_id, cj.attempt_count as job_attempt_count, + cj.max_attempts as job_max_attempts, cj.locked_at as job_locked_at + ) + select pd.job_id, pd.id, pd.import_batch_id, 'processing'::text, 'v3 enrichment claimed'::text, + 95::integer, null::text, pd.job_attempt_count, pd.job_max_attempts, pd.job_locked_at, + p_worker_id, + to_jsonb(pd.*) - 'job_id' - 'job_attempt_count' - 'job_max_attempts' - 'job_locked_at' + from patched_documents pd; +end; +$$; + +revoke execute on function public.claim_indexing_v3_agent_jobs(text, integer, integer) from public, anon, authenticated; +grant execute on function public.claim_indexing_v3_agent_jobs(text, integer, integer) to service_role; + +create or replace function public.update_indexing_v3_agent_job_status( + p_document_id uuid, p_status text, p_error text default null, p_next_run_at timestamptz default null +) +returns jsonb +language plpgsql +set search_path = public, extensions, pg_temp +as $$ +declare v_job_id uuid; +begin + if p_status not in ('pending', 'completed', 'failed', 'needs_enrichment_artifacts') then + raise exception 'invalid status %', p_status; + end if; + update public.indexing_v3_agent_jobs + set status = p_status, + enrichment_status = case + when p_status = 'completed' then 'completed' + when p_status = 'failed' then 'failed' + when p_status = 'needs_enrichment_artifacts' then 'needs_enrichment_artifacts' + else enrichment_status end, + last_error = p_error, + next_run_at = case when p_status = 'pending' then coalesce(p_next_run_at, now()) else null end, + locked_by = null, locked_at = null, updated_at = now() + where document_id = p_document_id + returning id into v_job_id; + return jsonb_build_object('ok', v_job_id is not null, 'job_id', v_job_id, + 'document_id', p_document_id, 'status', p_status); +end; +$$; + +revoke execute on function public.update_indexing_v3_agent_job_status(uuid, text, text, timestamptz) from public, anon, authenticated; +grant execute on function public.update_indexing_v3_agent_job_status(uuid, text, text, timestamptz) to service_role; + +create or replace function public.match_document_embedding_fields_text( + query_text text, match_count integer default 16, min_text_rank double precision default 0.0, + document_filters uuid[] default null, owner_filter uuid default null +) +returns table ( + id uuid, document_id uuid, source_chunk_id uuid, field_type text, content text, text_rank double precision +) +language sql stable set search_path = public, extensions, pg_temp +as $$ + with q as (select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq), + ranked as ( + select f.id, f.document_id, f.source_chunk_id, f.field_type, f.content, + ts_rank_cd(f.search_tsv, q.tsq)::double precision as text_rank + from public.document_embedding_fields f + join public.documents d on d.id = f.document_id + cross join q + where f.source_chunk_id is not null + and (document_filters is null or f.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' and f.search_tsv @@ q.tsq + ) + select * from ranked where text_rank >= min_text_rank + order by text_rank desc, id limit match_count; +$$; + +revoke execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) from public, anon, authenticated; +grant execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) to service_role; + +create table if not exists public.rag_visual_eval_cases ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + document_id uuid references public.documents(id) on delete set null, + case_name text not null, query text not null, + expected_unit_types text[] not null default '{}'::text[], + expected_terms text[] not null default '{}'::text[], + expected_image_type text, active boolean not null default true, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); +create index if not exists rag_visual_eval_cases_doc_idx on public.rag_visual_eval_cases(document_id, active); +create index if not exists rag_visual_eval_cases_owner_id_idx on public.rag_visual_eval_cases(owner_id); + +create table if not exists public.rag_visual_eval_runs ( + id uuid primary key default gen_random_uuid(), + case_id uuid not null references public.rag_visual_eval_cases(id) on delete cascade, + document_id uuid references public.documents(id) on delete set null, + passed boolean not null, top_hit boolean not null, matched_count integer not null default 0, + hit_payload jsonb not null default '{}'::jsonb, run_metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index if not exists rag_visual_eval_runs_case_id_idx on public.rag_visual_eval_runs(case_id); +create index if not exists rag_visual_eval_runs_document_id_idx on public.rag_visual_eval_runs(document_id); + +alter table public.rag_visual_eval_cases enable row level security; +alter table public.rag_visual_eval_runs enable row level security; +drop policy if exists "rag visual eval cases service role all" on public.rag_visual_eval_cases; +create policy "rag visual eval cases service role all" on public.rag_visual_eval_cases for all to service_role using (true) with check (true); +drop policy if exists "rag visual eval runs service role all" on public.rag_visual_eval_runs; +create policy "rag visual eval runs service role all" on public.rag_visual_eval_runs for all to service_role using (true) with check (true); +grant select, insert, update, delete on table public.rag_visual_eval_cases to service_role; +grant select, insert, update, delete on table public.rag_visual_eval_runs to service_role; diff --git a/supabase/schema.sql b/supabase/schema.sql index 6f1a9c6c3..5da342f1e 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -2880,6 +2880,76 @@ $$; revoke execute on function public.match_document_lookup_chunks_text(text, uuid[], integer, uuid) from public, anon, authenticated; grant execute on function public.match_document_lookup_chunks_text(text, uuid[], integer, uuid) to service_role; +create or replace function public.search_document_chunks( + p_document_id uuid, + p_query text, + match_count integer default 20, + p_owner_id uuid default null +) +returns table ( + id uuid, + page_number integer, + chunk_index integer, + section_heading text, + content text, + image_ids uuid[], + text_rank real, + trigram_score real +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with normalized as ( + select + websearch_to_tsquery('english', coalesce(p_query, '')) as query_tsv, + lower(trim(coalesce(p_query, ''))) as query_text + ), + tokens as ( + select distinct token + from normalized, + lateral regexp_split_to_table(normalized.query_text, '\s+') as token + where length(token) >= 3 + ) + select + c.id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.image_ids, + ts_rank_cd(c.search_tsv, normalized.query_tsv)::real as text_rank, + similarity(lower(coalesce(c.section_heading, '') || ' ' || c.content), normalized.query_text)::real as trigram_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join normalized + where c.document_id = p_document_id + and d.status = 'indexed' + and ( + (p_owner_id is null and d.owner_id is null) + or (p_owner_id is not null and (d.owner_id is null or d.owner_id = p_owner_id)) + ) + and ( + c.search_tsv @@ normalized.query_tsv + or lower(coalesce(c.section_heading, '') || ' ' || c.content) % normalized.query_text + or lower(coalesce(c.section_heading, '') || ' ' || c.content) like '%' || normalized.query_text || '%' + or exists ( + select 1 + from tokens t + where lower(coalesce(c.section_heading, '') || ' ' || c.content) like '%' || t.token || '%' + or lower(coalesce(c.section_heading, '') || ' ' || c.content) % t.token + ) + ) + order by + ts_rank_cd(c.search_tsv, normalized.query_tsv) desc, + similarity(lower(coalesce(c.section_heading, '') || ' ' || c.content), normalized.query_text) desc, + c.chunk_index asc + limit least(greatest(match_count, 1), 80); +$$; + +revoke execute on function public.search_document_chunks(uuid, text, integer, uuid) from public, anon, authenticated; +grant execute on function public.search_document_chunks(uuid, text, integer, uuid) to service_role; + create or replace function public.get_related_document_metadata( document_ids uuid[], owner_filter uuid default null @@ -3110,6 +3180,31 @@ as $$ limit match_count; $$; +create or replace function public.match_document_embedding_fields_text( + query_text text, match_count integer default 16, min_text_rank double precision default 0.0, + document_filters uuid[] default null, owner_filter uuid default null +) +returns table ( + id uuid, document_id uuid, source_chunk_id uuid, field_type text, content text, text_rank double precision +) +language sql stable set search_path = public, extensions, pg_temp +as $$ + with q as (select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq), + ranked as ( + select f.id, f.document_id, f.source_chunk_id, f.field_type, f.content, + ts_rank_cd(f.search_tsv, q.tsq)::double precision as text_rank + from public.document_embedding_fields f + join public.documents d on d.id = f.document_id + cross join q + where f.source_chunk_id is not null + and (document_filters is null or f.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and d.status = 'indexed' and f.search_tsv @@ q.tsq + ) + select * from ranked where text_rank >= min_text_rank + order by text_rank desc, id limit match_count; +$$; + create or replace view public.document_strict_gate_status with (security_invoker = true) as @@ -3710,6 +3805,8 @@ grant usage, select on all sequences in schema public to service_role; grant execute on all functions in schema public to service_role; revoke execute on function public.claim_indexing_v3_agent_jobs(text, integer, integer) from public, anon, authenticated; grant execute on function public.claim_indexing_v3_agent_jobs(text, integer, integer) to service_role; +revoke execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) from public, anon, authenticated; +grant execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) to service_role; revoke execute on function public.invoke_indexing_v3_agent(integer) from public, anon, authenticated; grant execute on function public.invoke_indexing_v3_agent(integer) to service_role; @@ -4172,6 +4269,41 @@ comment on index public.documents_indexing_v3_agent_claim_idx is comment on table public.indexing_v3_agent_jobs is 'Dedicated worker-state table for the v3 indexing / enrichment agent. Replaces JSONB state in documents.metadata. claim_indexing_v3_agent_jobs uses SKIP LOCKED here; update_indexing_v3_agent_job_status completes/fails a job. See migration 20260702190000 for transition notes.'; +create table if not exists public.rag_visual_eval_cases ( + id uuid primary key default gen_random_uuid(), + owner_id uuid references auth.users(id) on delete set null, + document_id uuid references public.documents(id) on delete set null, + case_name text not null, query text not null, + expected_unit_types text[] not null default '{}'::text[], + expected_terms text[] not null default '{}'::text[], + expected_image_type text, active boolean not null default true, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); +create index if not exists rag_visual_eval_cases_doc_idx on public.rag_visual_eval_cases(document_id, active); +create index if not exists rag_visual_eval_cases_owner_id_idx on public.rag_visual_eval_cases(owner_id); + +create table if not exists public.rag_visual_eval_runs ( + id uuid primary key default gen_random_uuid(), + case_id uuid not null references public.rag_visual_eval_cases(id) on delete cascade, + document_id uuid references public.documents(id) on delete set null, + passed boolean not null, top_hit boolean not null, matched_count integer not null default 0, + hit_payload jsonb not null default '{}'::jsonb, run_metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index if not exists rag_visual_eval_runs_case_id_idx on public.rag_visual_eval_runs(case_id); +create index if not exists rag_visual_eval_runs_document_id_idx on public.rag_visual_eval_runs(document_id); + +alter table public.rag_visual_eval_cases enable row level security; +alter table public.rag_visual_eval_runs enable row level security; +drop policy if exists "rag visual eval cases service role all" on public.rag_visual_eval_cases; +create policy "rag visual eval cases service role all" on public.rag_visual_eval_cases for all to service_role using (true) with check (true); +drop policy if exists "rag visual eval runs service role all" on public.rag_visual_eval_runs; +create policy "rag visual eval runs service role all" on public.rag_visual_eval_runs for all to service_role using (true) with check (true); +grant select, insert, update, delete on table public.rag_visual_eval_cases to service_role; +grant select, insert, update, delete on table public.rag_visual_eval_runs to service_role; + -- Curated clinical registry backing the Services and Forms modes: structured -- records (contacts, eligibility, referral pathways, criteria) for real WA -- entities, seeded from reviewed fixtures and linkable to verifying source diff --git a/tests/public-access-deep.test.ts b/tests/public-access-deep.test.ts index a5ad9ea60..881b64cf3 100644 --- a/tests/public-access-deep.test.ts +++ b/tests/public-access-deep.test.ts @@ -98,6 +98,7 @@ describe("public access deep checks", () => { NEXT_PUBLIC_SUPABASE_URL: "https://sjrfecxgysukkwxsowpy.supabase.co", SUPABASE_SERVICE_ROLE_KEY: "super-secret-service-role-key", OPENAI_API_KEY: "sk-super-secret-openai-key", + HEALTH_DEEP_PROBE_SECRET: undefined, }, isDemoMode: () => false, })); @@ -110,6 +111,7 @@ describe("public access deep checks", () => { expect(serialized).not.toContain("sk-super-secret-openai-key"); expect(body.checks.supabaseConfig).toBe("ok"); expect(body.checks.openaiConfig).toBe("ok"); + expect(body.checks.supabase).toBe("unauthorized"); }); }); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index b1d0d3c95..32144a386 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -104,6 +104,14 @@ const ragRetrievalLogsRetentionMigration = readFileSync( new URL("../supabase/migrations/20260702120000_rag_retrieval_logs_retention.sql", import.meta.url), "utf8", ).replace(/\s+/g, " "); +const liveDatabaseDriftMigration = readFileSync( + new URL("../supabase/migrations/20260705230000_reconcile_live_database_drift.sql", import.meta.url), + "utf8", +).replace(/\s+/g, " "); +const searchDocumentChunksOwnerScopeMigration = readFileSync( + new URL("../supabase/migrations/20260705133000_tighten_search_document_chunks_owner_scope.sql", import.meta.url), + "utf8", +).replace(/\s+/g, " "); function extractTextChunkFunction(sql: string) { const start = sql.indexOf("function public.match_document_chunks_text"); @@ -758,6 +766,22 @@ describe("Supabase schema Data API grants", () => { ); }); + it("reconciles live database drift for embedding-field text RPC and rag visual eval tables", () => { + for (const sql of [schema, liveDatabaseDriftMigration]) { + expect(sql).toContain("create or replace function public.match_document_embedding_fields_text"); + expect(sql).toContain("create table if not exists public.rag_visual_eval_cases"); + expect(sql).toContain("create table if not exists public.rag_visual_eval_runs"); + expect(sql).toContain('create policy "rag visual eval cases service role all"'); + expect(sql).toContain('create policy "rag visual eval runs service role all"'); + expect(sql).toContain( + "revoke execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) from public, anon, authenticated", + ); + expect(sql).toContain( + "grant execute on function public.match_document_embedding_fields_text(text, integer, double precision, uuid[], uuid) to service_role", + ); + } + }); + it("reconciles search_schema_health index drift with canonical creates and live aliases", () => { expect(searchHealthIndexesMigration).toContain("create index if not exists documents_title_trgm_idx"); expect(searchHealthIndexesMigration).toContain("create index if not exists document_labels_label_trgm_idx"); @@ -769,6 +793,14 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("jsonb_array_elements_text(index_aliases -> index_name)"); }); + it("mirrors tightened search_document_chunks owner scope in schema and migration", () => { + expect(searchDocumentChunksOwnerScopeMigration).toContain("(p_owner_id is null and d.owner_id is null)"); + expect(schema).toContain("create or replace function public.search_document_chunks("); + expect(schema).toContain( + "revoke execute on function public.search_document_chunks(uuid, text, integer, uuid) from public, anon, authenticated", + ); + }); + it("surfaces stale commit generation RPCs through search_schema_health", () => { for (const sql of [schema, searchSchemaHealthM13GuardMigration]) { expect(sql).toContain("commit_fn_def := pg_get_functiondef(");