From 9f3cb960fdef7f0d694fa0077674849648ef6c0f Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:39:09 +0800 Subject: [PATCH] perf(db): optimize scoped retrieval queries --- docs/branch-review-ledger.md | 1 + scripts/generate-drift-manifest.ts | 22 +- supabase/drift-manifest.json | 26 +- ...160000_optimize_owner_public_retrieval.sql | 366 +++++++++++++++++ supabase/schema.sql | 373 ++++++++++++++++-- tests/retrieval-access-scope.test.ts | 62 +++ 6 files changed, 794 insertions(+), 56 deletions(-) create mode 100644 supabase/migrations/20260717160000_optimize_owner_public_retrieval.sql diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 4b668c49e..ed4281b39 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -566,3 +566,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-17 | codex/pwa-privacy-safe-20260717 | 35fa8c929d44a9bd84b3f7f2b795354d3b6dae02 | privacy-safe PWA shell and merge-readiness review | No remaining high-confidence product defect in the changed scope. The pre-push browser gate found and fixed one P2 test defect: cleanup referenced `PWA_CACHE_PREFIX` without passing it into the browser context, and the cold installability flow now has a focused 120-second budget. The worker caches only the generic offline page and allow-listed public shell assets; navigations, APIs, auth, queries, documents, uploads, signed URLs, range requests, and cross-origin traffic remain network-only. | Current-main integration; focused Vitest 81/81; full uncached ESLint; TypeScript; scoped Prettier and diff checks; production Webpack build generated 1,043 pages and the client-bundle secret scan passed; full Vitest produced 2,506 passes plus six contention timeouts, with all affected files passing 24/24 serially; focused Chromium PWA 2/2. No Supabase/OpenAI/live-provider checks run. | | 2026-07-17 | codex/historical-branch-cleanup-20260717 | e36ac0c6628264c7ed6c494a597a62d0214b68f6 | branch-cleanup and historical-content recovery | Completed the pending historical cleanup: deleted 55 exact-SHA remote refs and 20 redundant local refs, removed nine clean merged worktrees, preserved every dirty, active, open-PR, or patch-unique worktree, and recovered the still-useful governance incident runbook from `codex/domain-1-governance-remediation`. Historical code changes were either tied to merged PRs or reviewed as superseded by current implementations; open PRs #699, #700, #702, and #704 remain protected. | Fresh `git fetch --prune`; GitHub PR inventory and exact commit-to-merged-PR associations; exact remote SHA rechecks before deletion; cherry-pick-aware logs; two-dot tree and branch-only-file review; Codex task-to-worktree cross-check; focused documentation validation recorded in the cleanup PR. No OpenAI, Supabase, production-data, or live clinical workflow was run. | | 2026-07-17 | PR #704 / codex/scroll-geometry-stability-20260717 | 35e74ddbd61bacc5b34f06efbd58091f092665fd | nested scroll-source review follow-up | Confirmed the outside-diff CodeRabbit finding: the standalone shell shared one intent history across main and descendant scroll containers, so a switch from a deep main offset to a near-zero nested offset could falsely reveal chrome. Scroll metrics now identify their source, source changes rebase direction and travel while preserving visibility, and unit/UI regressions cover the switch. No unresolved actionable review finding remains. | Focused Vitest 9/9; TypeScript; scoped ESLint; Prettier; `git diff --check`. Exact-head hosted CI and UI remain required after push. No Supabase/OpenAI/live-provider checks run. | +| 2026-07-17 | codex/chat-supabase-migration-preflight-b463 | f7c4e293ef35acc54f2b82bbccb2990d51289d5c | live production Supabase security, integrity, drift, and performance review plus remediation | Resolved and deployed the P2 retrieval-performance issue with `20260717160000_optimize_owner_public_retrieval`: owner/public filtering now happens in one scoped query and index-unit text/term candidates use separate GIN-friendly branches. Warm text retrieval improved from 1.269 seconds to 34 ms; warm index-unit retrieval completed in 36-39 ms (first cold run 2.376 seconds with 2,009 physical reads). No P0/P1 security, privacy, RLS, privilege, storage, migration-history, or integrity issue was found. | Isolated Docker schema replay; pre-deploy drift showed exactly four intended function changes; linked production push; post-deploy `No unexpected schema drift`; exact project and migration-history checks; security and performance advisors; live access-scope parity; bounded `EXPLAIN ANALYZE`; post-migration logs; focused Vitest 74/74; offline RAG fixtures 36/36 and contract tests 291/291; ESLint; TypeScript; function grants; production readiness. Full `verify:cheap` reached the 10-minute host timeout during broad Vitest and ended with EPIPE; focused and domain checks passed. No OpenAI calls, write load test, or backup/PITR restore test was performed. | diff --git a/scripts/generate-drift-manifest.ts b/scripts/generate-drift-manifest.ts index 2c9efddb4..55deb7fcb 100644 --- a/scripts/generate-drift-manifest.ts +++ b/scripts/generate-drift-manifest.ts @@ -18,11 +18,12 @@ import { fileURLToPath } from "node:url"; * Flags: * --keep leave the container running (for inspection / DR rehearsal) * --port host port for the scratch Postgres (default 56599) + * --container override the scratch container name for concurrent worktrees * --image override the Postgres image tag */ const IMAGE_DEFAULT = "supabase/postgres:17.6.1.127"; -const CONTAINER = "clinical-kb-drift-manifest"; +const CONTAINER_DEFAULT = "clinical-kb-drift-manifest"; const repoUrl = (relative: string) => fileURLToPath(new URL(`../${relative}`, import.meta.url)); @@ -43,6 +44,7 @@ function docker(args: string[], input?: string): string { async function main() { const image = arg("--image") ?? IMAGE_DEFAULT; const port = arg("--port") ?? "56599"; + const container = arg("--container") ?? CONTAINER_DEFAULT; const keep = process.argv.includes("--keep"); try { @@ -57,15 +59,15 @@ async function main() { const scaffoldSql = readFileSync(repoUrl("scripts/sql/drift-replay-scaffold.sql"), "utf8"); const { normalizedSchemaSha256 } = await import("./check-drift"); - console.log(`Starting scratch container ${CONTAINER} (${image}) on port ${port}…`); + console.log(`Starting scratch container ${container} (${image}) on port ${port}…`); try { - docker(["rm", "-f", CONTAINER]); + docker(["rm", "-f", container]); } catch { // not running — fine } const startedAt = Date.now(); const scratchPassword = randomBytes(16).toString("hex"); - docker(["run", "-d", "--name", CONTAINER, "-e", `POSTGRES_PASSWORD=${scratchPassword}`, "-p", `${port}:5432`, image]); + docker(["run", "-d", "--name", container, "-e", `POSTGRES_PASSWORD=${scratchPassword}`, "-p", `${port}:5432`, image]); try { // The Supabase image briefly accepts connections before its init migrations @@ -74,7 +76,7 @@ async function main() { let consecutiveReadyChecks = 0; for (let attempt = 0; attempt < 60; attempt += 1) { try { - docker(["exec", CONTAINER, "pg_isready", "-U", "postgres", "-q"]); + docker(["exec", container, "pg_isready", "-U", "postgres", "-q"]); consecutiveReadyChecks += 1; if (consecutiveReadyChecks >= 5) break; } catch { @@ -88,7 +90,7 @@ async function main() { const psql = (user: string, sql: string) => docker( - ["exec", "-i", CONTAINER, "psql", "-U", user, "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-q", "-f", "-"], + ["exec", "-i", container, "psql", "-U", user, "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-q", "-f", "-"], sql, ); @@ -100,7 +102,7 @@ async function main() { console.log(`Replay complete in ${replaySeconds}s (container start included).`); const raw = docker( - ["exec", "-i", CONTAINER, "psql", "-U", "postgres", "-d", "postgres", "-tA", "-v", "ON_ERROR_STOP=1", "-f", "-"], + ["exec", "-i", container, "psql", "-U", "postgres", "-d", "postgres", "-tA", "-v", "ON_ERROR_STOP=1", "-f", "-"], "select public.schema_drift_snapshot()::text;", ).trim(); const snapshot = JSON.parse(raw) as Record; @@ -119,12 +121,12 @@ async function main() { console.log("Next: run `npm run check:drift` against live (needs service-role env)."); } finally { if (keep) { - console.log(`Container ${CONTAINER} kept running on port ${port} (--keep).`); + console.log(`Container ${container} kept running on port ${port} (--keep).`); } else { try { - docker(["rm", "-f", CONTAINER]); + docker(["rm", "-f", container]); } catch { - console.warn(`Could not remove container ${CONTAINER}; remove it manually.`); + console.warn(`Could not remove container ${container}; remove it manually.`); } } } diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index c54ee62fd..44f0b66f7 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-07-17T06:06:10.141Z", + "generated_at": "2026-07-17T11:30:04.370Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127", - "schema_sha256": "b9faccf4dac098def5d9547ca1b4cf0bb24621f1b76e9aed1b57d2495b65d72b", - "replay_seconds": 60, + "schema_sha256": "59ee18d0723774416b931bea9f4fef96fa699eb99ba61d218b6870a621dde649", + "replay_seconds": 45, "snapshot": { "views": [ { @@ -6481,7 +6481,15 @@ "postgres=X/postgres", "service_role=X/postgres" ], - "def_hash": "698d24f90e3fa5f17ef5898e3ba2d8ff", + "def_hash": "2b890b6f6c617f78f41f06293c9f7ade", + "signature": "public.match_document_chunks_text_scoped(text,integer,uuid[],uuid,boolean)" + }, + { + "acl": [ + "postgres=X/postgres", + "service_role=X/postgres" + ], + "def_hash": "4bf81d62c0953a8d11c80c37de6045e7", "signature": "public.match_document_chunks_text_v2(text,integer,uuid[],uuid,boolean)" }, { @@ -6537,7 +6545,15 @@ "postgres=X/postgres", "service_role=X/postgres" ], - "def_hash": "7e7e5a83bad724a62fdc66a0213b843b", + "def_hash": "e802785a1872f92ca4b748c08c57435d", + "signature": "public.match_document_index_units_hybrid_scoped(extensions.vector,text,integer,double precision,uuid[],uuid,boolean)" + }, + { + "acl": [ + "postgres=X/postgres", + "service_role=X/postgres" + ], + "def_hash": "38282636c4c22a4f01974a943aa20d42", "signature": "public.match_document_index_units_hybrid_v2(extensions.vector,text,integer,double precision,uuid[],uuid,boolean)" }, { diff --git a/supabase/migrations/20260717160000_optimize_owner_public_retrieval.sql b/supabase/migrations/20260717160000_optimize_owner_public_retrieval.sql new file mode 100644 index 000000000..6f501c206 --- /dev/null +++ b/supabase/migrations/20260717160000_optimize_owner_public_retrieval.sql @@ -0,0 +1,366 @@ +-- Remove duplicate owner/public execution from the two retrieval RPCs observed +-- as production latency hotspots on 2026-07-17. The scoped helpers evaluate +-- the complete access predicate inside one query, preserving fail-closed owner +-- semantics while allowing public rows when explicitly requested. + +create or replace function public.match_document_chunks_text_scoped( + query_text text, + match_count integer, + document_filters uuid[], + owner_filter uuid, + include_public boolean +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + lexical_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + -- Keep the chunk/title probes separate so each GIN index remains usable. + -- Apply access and committed-generation gates before candidate union/ranking. + chunk_hits as ( + select c.id + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where c.search_tsv @@ query.tsq + and (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.index_generation_id) + ), + title_chunk_hits as ( + select c.id + from public.documents d + cross join query + join public.document_chunks c on c.document_id = d.id + where d.title_search_tsv @@ query.tsq + and (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.index_generation_id) + ), + lexical_candidates as ( + select chunk_hits.id from chunk_hits + union + select title_chunk_hits.id from title_chunk_hits + ), + ranked as ( + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank + from lexical_candidates cand + join public.document_chunks c on c.id = cand.id + join public.documents d on d.id = c.document_id + cross join query + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit least(greatest(match_count * 2, 24), 96) + ), + doc_labels as ( + select + l.document_id, + coalesce( + jsonb_agg( + jsonb_build_object( + 'id', l.id, + 'document_id', l.document_id, + 'owner_id', l.owner_id, + 'label', l.label, + 'label_type', l.label_type, + 'source', l.source, + 'confidence', l.confidence, + 'metadata', l.metadata, + 'created_at', l.created_at, + 'updated_at', l.updated_at + ) + order by l.confidence desc, l.label + ), + '[]'::jsonb + ) as labels + from public.document_labels l + where l.document_id in (select distinct ranked.document_id from ranked) + and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' + and coalesce(l.metadata->>'hidden', 'false') <> 'true' + group by l.document_id + ), + doc_summaries as ( + select distinct on (s.document_id) + s.document_id, + s.summary + from public.document_summaries s + where s.document_id in (select distinct ranked.document_id from ranked) + order by s.document_id + ) + select + ranked.id, + ranked.document_id, + ranked.title, + ranked.file_name, + ranked.page_number, + ranked.chunk_index, + ranked.section_heading, + ranked.content, + ranked.retrieval_synopsis, + ranked.image_ids, + ranked.source_metadata, + coalesce(doc_labels.labels, '[]'::jsonb) as document_labels, + doc_summaries.summary as document_summary, + 0::double precision as similarity, + ranked.text_rank, + least(0.5, 0.18 + (least(ranked.text_rank, 1) * 0.3))::double precision as hybrid_score, + least(0.99, 0.4 + (least(ranked.text_rank, 1) * 0.59))::double precision as lexical_score, + public.chunk_image_metadata(ranked.image_ids) as images + from ranked + left join doc_labels on doc_labels.document_id = ranked.document_id + left join doc_summaries on doc_summaries.document_id = ranked.document_id + order by hybrid_score desc, text_rank desc, ranked.id + limit match_count; +$$; + +create or replace function public.match_document_index_units_hybrid_scoped( + query_embedding extensions.vector(1536), + query_text text, + match_count integer, + min_similarity double precision, + document_filters uuid[], + owner_filter uuid, + include_public boolean +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +set plan_cache_mode = 'force_custom_plan' +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + -- Split the OR into separately indexable GIN probes. Both branches enforce + -- the full access/generation scope before their ids can enter the union. + text_hits as ( + select u.id + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where u.search_tsv @@ query.tsq + and d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and public.is_committed_artifact_generation(u.metadata, d.metadata) + and u.source_chunk_id is not null + ), + term_hits as ( + select u.id + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where u.normalized_terms && query.terms + and d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and public.is_committed_artifact_generation(u.metadata, d.metadata) + and u.source_chunk_id is not null + ), + candidate_ids as ( + select text_hits.id from text_hits + union + select term_hits.id from term_hits + ), + ranked as ( + select + u.id, + u.document_id, + u.source_chunk_id, + u.source_image_id, + u.unit_type, + u.title, + u.content, + u.page_start, + u.page_end, + u.heading_path, + u.normalized_terms, + u.source_span, + u.quality_score, + u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + ( + ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case + when u.unit_type in ( + 'askable_question', 'table_fact', 'clinical_fact', 'threshold', + 'workflow_step', 'medication_monitoring', 'alias', 'visual_summary', + 'flowchart_step', 'diagram_decision', 'risk_matrix_cell', + 'medication_chart_row', 'chart_finding', 'visual_askable_question', + 'table_threshold' + ) then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 + end + )::double precision as text_rank, + u.metadata + from candidate_ids candidates + join public.document_index_units u on u.id = candidates.id + cross join query + order by text_rank desc + limit greatest(match_count * 3, 48) + ) + select + id, + document_id, + source_chunk_id, + source_image_id, + unit_type, + title, + content, + page_start, + page_end, + heading_path, + normalized_terms, + source_span, + quality_score, + extraction_mode, + similarity, + text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case + when unit_type in ('askable_question', 'threshold', 'table_fact', 'table_threshold', 'visual_askable_question') then 0.04 + when unit_type in ('workflow_step', 'medication_monitoring', 'flowchart_step', 'diagram_decision', 'medication_chart_row', 'risk_matrix_cell') then 0.03 + else 0 + end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, id + limit match_count; +$$; + +-- The versioned RPCs now delegate once to the scoped helpers. This replaces +-- the previous owner query UNION ALL public query pattern. +create or replace function public.match_document_chunks_text_v2( + query_text text, + match_count integer default 12, + document_filters uuid[] default null, + owner_filter uuid default '00000000-0000-0000-0000-000000000000'::uuid, + include_public boolean default true +) +returns table ( + id uuid, document_id uuid, title text, file_name text, page_number integer, + chunk_index integer, section_heading text, content text, retrieval_synopsis text, + image_ids uuid[], source_metadata jsonb, document_labels jsonb, document_summary text, + similarity double precision, text_rank double precision, hybrid_score double precision, + lexical_score double precision, images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select * + from public.match_document_chunks_text_scoped($1, $2, $3, $4, $5); +$$; + +create or replace function public.match_document_index_units_hybrid_v2( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default '00000000-0000-0000-0000-000000000000'::uuid, + include_public boolean default true +) +returns table ( + id uuid, document_id uuid, source_chunk_id uuid, source_image_id uuid, unit_type text, title text, + content text, page_start integer, page_end integer, heading_path text[], normalized_terms text[], + source_span jsonb, quality_score real, extraction_mode text, similarity double precision, + text_rank double precision, hybrid_score double precision, metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select * + from public.match_document_index_units_hybrid_scoped($1, $2, $3, $4, $5, $6, $7); +$$; + +revoke all on function public.match_document_chunks_text_scoped(text, integer, uuid[], uuid, boolean) + from public, anon, authenticated; +grant execute on function public.match_document_chunks_text_scoped(text, integer, uuid[], uuid, boolean) + to service_role; +revoke all on function public.match_document_index_units_hybrid_scoped( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) from public, anon, authenticated; +grant execute on function public.match_document_index_units_hybrid_scoped( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) to service_role; + +revoke all on function public.match_document_chunks_text_v2(text, integer, uuid[], uuid, boolean) + from public, anon, authenticated; +grant execute on function public.match_document_chunks_text_v2(text, integer, uuid[], uuid, boolean) + to service_role; +revoke all on function public.match_document_index_units_hybrid_v2( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) from public, anon, authenticated; +grant execute on function public.match_document_index_units_hybrid_v2( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) to service_role; diff --git a/supabase/schema.sql b/supabase/schema.sql index 163495444..292bfeb9f 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -6966,42 +6966,321 @@ create or replace function public.corpus_topic_term_stats_v2( from combined group by term order by term; $$; +-- Single-pass scoped implementations for the production retrieval hotspots. +-- Access and generation predicates are applied before candidate ranking. +create or replace function public.match_document_chunks_text_scoped( + query_text text, + match_count integer, + document_filters uuid[], + owner_filter uuid, + include_public boolean +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + lexical_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + -- Keep the chunk/title probes separate so each GIN index remains usable. + -- Apply access and committed-generation gates before candidate union/ranking. + chunk_hits as ( + select c.id + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where c.search_tsv @@ query.tsq + and (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.index_generation_id) + ), + title_chunk_hits as ( + select c.id + from public.documents d + cross join query + join public.document_chunks c on c.document_id = d.id + where d.title_search_tsv @@ query.tsq + and (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.index_generation_id) + ), + lexical_candidates as ( + select chunk_hits.id from chunk_hits + union + select title_chunk_hits.id from title_chunk_hits + ), + ranked as ( + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank + from lexical_candidates cand + join public.document_chunks c on c.id = cand.id + join public.documents d on d.id = c.document_id + cross join query + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit least(greatest(match_count * 2, 24), 96) + ), + doc_labels as ( + select + l.document_id, + coalesce( + jsonb_agg( + jsonb_build_object( + 'id', l.id, + 'document_id', l.document_id, + 'owner_id', l.owner_id, + 'label', l.label, + 'label_type', l.label_type, + 'source', l.source, + 'confidence', l.confidence, + 'metadata', l.metadata, + 'created_at', l.created_at, + 'updated_at', l.updated_at + ) + order by l.confidence desc, l.label + ), + '[]'::jsonb + ) as labels + from public.document_labels l + where l.document_id in (select distinct ranked.document_id from ranked) + and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' + and coalesce(l.metadata->>'hidden', 'false') <> 'true' + group by l.document_id + ), + doc_summaries as ( + select distinct on (s.document_id) + s.document_id, + s.summary + from public.document_summaries s + where s.document_id in (select distinct ranked.document_id from ranked) + order by s.document_id + ) + select + ranked.id, + ranked.document_id, + ranked.title, + ranked.file_name, + ranked.page_number, + ranked.chunk_index, + ranked.section_heading, + ranked.content, + ranked.retrieval_synopsis, + ranked.image_ids, + ranked.source_metadata, + coalesce(doc_labels.labels, '[]'::jsonb) as document_labels, + doc_summaries.summary as document_summary, + 0::double precision as similarity, + ranked.text_rank, + least(0.5, 0.18 + (least(ranked.text_rank, 1) * 0.3))::double precision as hybrid_score, + least(0.99, 0.4 + (least(ranked.text_rank, 1) * 0.59))::double precision as lexical_score, + public.chunk_image_metadata(ranked.image_ids) as images + from ranked + left join doc_labels on doc_labels.document_id = ranked.document_id + left join doc_summaries on doc_summaries.document_id = ranked.document_id + order by hybrid_score desc, text_rank desc, ranked.id + limit match_count; +$$; + +create or replace function public.match_document_index_units_hybrid_scoped( + query_embedding extensions.vector(1536), + query_text text, + match_count integer, + min_similarity double precision, + document_filters uuid[], + owner_filter uuid, + include_public boolean +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +set plan_cache_mode = 'force_custom_plan' +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + -- Split the OR into separately indexable GIN probes. Both branches enforce + -- the full access/generation scope before their ids can enter the union. + text_hits as ( + select u.id + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where u.search_tsv @@ query.tsq + and d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and public.is_committed_artifact_generation(u.metadata, d.metadata) + and u.source_chunk_id is not null + ), + term_hits as ( + select u.id + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where u.normalized_terms && query.terms + and d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public) + and public.is_committed_artifact_generation(u.metadata, d.metadata) + and u.source_chunk_id is not null + ), + candidate_ids as ( + select text_hits.id from text_hits + union + select term_hits.id from term_hits + ), + ranked as ( + select + u.id, + u.document_id, + u.source_chunk_id, + u.source_image_id, + u.unit_type, + u.title, + u.content, + u.page_start, + u.page_end, + u.heading_path, + u.normalized_terms, + u.source_span, + u.quality_score, + u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + ( + ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case + when u.unit_type in ( + 'askable_question', 'table_fact', 'clinical_fact', 'threshold', + 'workflow_step', 'medication_monitoring', 'alias', 'visual_summary', + 'flowchart_step', 'diagram_decision', 'risk_matrix_cell', + 'medication_chart_row', 'chart_finding', 'visual_askable_question', + 'table_threshold' + ) then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 + end + )::double precision as text_rank, + u.metadata + from candidate_ids candidates + join public.document_index_units u on u.id = candidates.id + cross join query + order by text_rank desc + limit greatest(match_count * 3, 48) + ) + select + id, + document_id, + source_chunk_id, + source_image_id, + unit_type, + title, + content, + page_start, + page_end, + heading_path, + normalized_terms, + source_span, + quality_score, + extraction_mode, + similarity, + text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case + when unit_type in ('askable_question', 'threshold', 'table_fact', 'table_threshold', 'visual_askable_question') then 0.04 + when unit_type in ('workflow_step', 'medication_monitoring', 'flowchart_step', 'diagram_decision', 'medication_chart_row', 'risk_matrix_cell') then 0.03 + else 0 + end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, id + limit match_count; +$$; + create or replace function public.match_document_chunks_text_v2( - query_text text, match_count integer default 12, document_filters uuid[] default null, + query_text text, + match_count integer default 12, + document_filters uuid[] default null, owner_filter uuid default '00000000-0000-0000-0000-000000000000'::uuid, include_public boolean default true -) returns table ( +) +returns table ( id uuid, document_id uuid, title text, file_name text, page_number integer, chunk_index integer, section_heading text, content text, retrieval_synopsis text, image_ids uuid[], source_metadata jsonb, document_labels jsonb, document_summary text, similarity double precision, text_rank double precision, hybrid_score double precision, lexical_score double precision, images jsonb -) language sql stable set search_path = public, extensions, pg_temp as $$ - with combined as ( - select hit.id, hit.document_id, hit.title, hit.file_name, hit.page_number, hit.chunk_index, - hit.section_heading, hit.content, hit.retrieval_synopsis, hit.image_ids, hit.source_metadata, - hit.document_labels, hit.document_summary, hit.similarity, hit.text_rank, hit.hybrid_score, - coalesce(nullif(to_jsonb(hit)->>'lexical_score', '')::double precision, hit.hybrid_score) as lexical_score, - hit.images - from public.match_document_chunks_text($1, $2, $3, $4) hit - union all - select hit.id, hit.document_id, hit.title, hit.file_name, hit.page_number, hit.chunk_index, - hit.section_heading, hit.content, hit.retrieval_synopsis, hit.image_ids, hit.source_metadata, - hit.document_labels, hit.document_summary, hit.similarity, hit.text_rank, hit.hybrid_score, - coalesce(nullif(to_jsonb(hit)->>'lexical_score', '')::double precision, hit.hybrid_score) as lexical_score, - hit.images - from public.match_document_chunks_text($1, $2, $3, '00000000-0000-0000-0000-000000000000'::uuid) hit - where $5 and $4 <> '00000000-0000-0000-0000-000000000000'::uuid - ), deduped as ( - select *, row_number() over (partition by id order by hybrid_score desc, text_rank desc) as access_rank - from combined - ) - select id, document_id, title, file_name, page_number, chunk_index, section_heading, content, - retrieval_synopsis, image_ids, source_metadata, document_labels, document_summary, - similarity, text_rank, hybrid_score, lexical_score, images - from deduped where access_rank = 1 - order by hybrid_score desc, text_rank desc, id - limit greatest(1, least($2, 100)); +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select * + from public.match_document_chunks_text_scoped($1, $2, $3, $4, $5); $$; create or replace function public.match_document_chunks_hybrid_v2( @@ -7159,25 +7438,26 @@ create or replace function public.match_document_embedding_fields_hybrid_v2( $$; create or replace function public.match_document_index_units_hybrid_v2( - query_embedding extensions.vector(1536), query_text text, match_count integer default 24, - min_similarity double precision default 0.1, document_filters uuid[] default null, + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, owner_filter uuid default '00000000-0000-0000-0000-000000000000'::uuid, include_public boolean default true -) returns table ( +) +returns table ( id uuid, document_id uuid, source_chunk_id uuid, source_image_id uuid, unit_type text, title text, content text, page_start integer, page_end integer, heading_path text[], normalized_terms text[], source_span jsonb, quality_score real, extraction_mode text, similarity double precision, text_rank double precision, hybrid_score double precision, metadata jsonb -) language sql stable set search_path = public, extensions, pg_temp as $$ - with combined as ( - select * from public.match_document_index_units_hybrid($1, $2, $3, $4, $5, $6) - union all - select * from public.match_document_index_units_hybrid($1, $2, $3, $4, $5, '00000000-0000-0000-0000-000000000000'::uuid) - where $7 and $6 <> '00000000-0000-0000-0000-000000000000'::uuid - ), ranked as (select *, row_number() over (partition by id order by hybrid_score desc) access_rank from combined) - select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, - heading_path, normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, hybrid_score, metadata - from ranked where access_rank = 1 order by hybrid_score desc, id limit greatest(1, least($3, 100)); +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select * + from public.match_document_index_units_hybrid_scoped($1, $2, $3, $4, $5, $6, $7); $$; create or replace function public.match_document_memory_cards_hybrid_v3( @@ -7201,6 +7481,17 @@ create or replace function public.match_document_memory_cards_hybrid_v3( from ranked where access_rank = 1 order by hybrid_score desc, rrf_score desc, id limit greatest(1, least($3, 100)); $$; +revoke all on function public.match_document_chunks_text_scoped(text, integer, uuid[], uuid, boolean) + from public, anon, authenticated; +grant execute on function public.match_document_chunks_text_scoped(text, integer, uuid[], uuid, boolean) + to service_role; +revoke all on function public.match_document_index_units_hybrid_scoped( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) from public, anon, authenticated; +grant execute on function public.match_document_index_units_hybrid_scoped( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean +) to service_role; + revoke all on function public.retrieval_owner_matches_v2(uuid, uuid, boolean) from public, anon, authenticated; revoke all on function public.corpus_topic_term_stats_v2(text[], uuid, boolean) from public, anon, authenticated; revoke all on function public.match_document_chunks_text_v2(text, integer, uuid[], uuid, boolean) from public, anon, authenticated; diff --git a/tests/retrieval-access-scope.test.ts b/tests/retrieval-access-scope.test.ts index be17977a2..0e224333b 100644 --- a/tests/retrieval-access-scope.test.ts +++ b/tests/retrieval-access-scope.test.ts @@ -4,6 +4,19 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; const root = process.cwd(); +const optimizedRetrievalMigration = readFileSync( + join(root, "supabase/migrations/20260717160000_optimize_owner_public_retrieval.sql"), + "utf8", +); +const canonicalSchema = readFileSync(join(root, "supabase/schema.sql"), "utf8"); + +function between(source: string, start: string, end: string): string { + const startIndex = source.indexOf(start); + const endIndex = source.indexOf(end, startIndex + start.length); + expect(startIndex).toBeGreaterThanOrEqual(0); + expect(endIndex).toBeGreaterThan(startIndex); + return source.slice(startIndex, endIndex); +} describe("owner-plus-public retrieval contract", () => { it("threads the canonical access scope through cache and retrieval contracts", () => { @@ -49,6 +62,55 @@ describe("owner-plus-public retrieval contract", () => { expect(migration).not.toContain("returns jsonb"); }); + it("executes the production text and index-unit hotspots as one scoped query", () => { + for (const source of [optimizedRetrievalMigration, canonicalSchema]) { + const scopedText = between( + source, + "create or replace function public.match_document_chunks_text_scoped(\n query_text text,\n match_count integer,", + "create or replace function public.match_document_index_units_hybrid_scoped(\n query_embedding extensions.vector(1536),\n query_text text,\n match_count integer,", + ); + expect(scopedText).toContain("chunk_hits as ("); + expect(scopedText).toContain("title_chunk_hits as ("); + expect(scopedText).toContain("public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public)"); + expect(scopedText).toContain( + "public.is_committed_document_generation(c.index_generation_id, d.index_generation_id)", + ); + + const scopedIndexUnits = between( + source, + "create or replace function public.match_document_index_units_hybrid_scoped(\n query_embedding extensions.vector(1536),\n query_text text,\n match_count integer,", + "create or replace function public.match_document_chunks_text_v2(", + ); + expect(scopedIndexUnits).toContain("text_hits as ("); + expect(scopedIndexUnits).toContain("term_hits as ("); + expect(scopedIndexUnits).toContain("select text_hits.id from text_hits\n union"); + expect(scopedIndexUnits).not.toContain("u.search_tsv @@ query.tsq or u.normalized_terms && query.terms"); + expect(scopedIndexUnits).toContain("public.retrieval_owner_matches_v2(owner_filter, d.owner_id, include_public)"); + + const textWrapper = between( + source, + "create or replace function public.match_document_chunks_text_v2(", + source === canonicalSchema + ? "create or replace function public.match_document_chunks_hybrid_v2(" + : "create or replace function public.match_document_index_units_hybrid_v2(", + ); + expect(textWrapper).toContain("from public.match_document_chunks_text_scoped($1, $2, $3, $4, $5)"); + expect(textWrapper).not.toContain("union all"); + + const indexWrapper = between( + source, + "create or replace function public.match_document_index_units_hybrid_v2(", + source === canonicalSchema + ? "create or replace function public.match_document_memory_cards_hybrid_v3(" + : "revoke all on function public.match_document_chunks_text_scoped(", + ); + expect(indexWrapper).toContain( + "from public.match_document_index_units_hybrid_scoped($1, $2, $3, $4, $5, $6, $7)", + ); + expect(indexWrapper).not.toContain("union all"); + } + }); + it("merges exact-owner and public rows when a versioned RPC is not deployed yet", async () => { const { callVersionedRetrievalRpc } = await import("../src/lib/rag"); const rpc = vi.fn(async (name: string, args: Record) => {