diff --git a/package.json b/package.json index aaaeefd4a..62d96ba7c 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "governance:release": "npm run check:document-label-coverage && npm run check:document-label-governance && npm run audit:source-governance:release", "check:document-label-coverage": "tsx scripts/check-document-label-coverage.ts", "check:document-label-governance": "tsx scripts/check-document-label-governance.ts", + "check:retrieval-owner-migration": "tsx scripts/check-retrieval-owner-migration.ts", "backfill:gold-labels": "tsx scripts/backfill-gold-document-labels.ts", "backfill:smart-v2-labels": "npm run classify:documents -- --all-owners --limit 5000 --only-missing-smart-v2", "tags:backfill": "tsx scripts/backfill-document-tags.ts", diff --git a/scripts/check-retrieval-owner-migration.ts b/scripts/check-retrieval-owner-migration.ts new file mode 100644 index 000000000..632fd5c01 --- /dev/null +++ b/scripts/check-retrieval-owner-migration.ts @@ -0,0 +1,41 @@ +import { createAdminClient } from "@/lib/supabase/admin"; + +const SENTINEL = "00000000-0000-0000-0000-000000000000"; + +async function main() { + const supabase = createAdminClient(); + const { data, error } = await supabase.rpc("search_schema_health"); + if (error) { + console.error("[Retrieval Owner Migration] FAIL: search_schema_health unavailable:", error.message); + process.exit(1); + } + + const { data: sentinelCheck, error: sentinelError } = await supabase.rpc( + "retrieval_owner_matches" as never, + { + owner_filter: SENTINEL, + row_owner_id: null, + } as never, + ); + + if (sentinelError) { + console.error( + "[Retrieval Owner Migration] FAIL: retrieval_owner_matches RPC missing or broken:", + sentinelError.message, + ); + process.exit(1); + } + + if (sentinelCheck !== true) { + console.error("[Retrieval Owner Migration] FAIL: sentinel did not match public owner_id IS NULL."); + process.exit(1); + } + + console.log("[Retrieval Owner Migration] PASS: retrieval_owner_matches sentinel is live."); + console.log("[Retrieval Owner Migration] search_schema_health:", JSON.stringify(data)); +} + +main().catch((error) => { + console.error("[Retrieval Owner Migration] FAIL:", error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/lib/deep-memory.ts b/src/lib/deep-memory.ts index fe6d02622..c4b1130a7 100644 --- a/src/lib/deep-memory.ts +++ b/src/lib/deep-memory.ts @@ -1,7 +1,7 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { buildClinicalTextSearchQuery, classifyRagQuery, normalizedClinicalSearchTokens } from "@/lib/clinical-search"; import { logger } from "@/lib/logger"; -import { requireOwnerScope } from "@/lib/owner-scope"; +import { retrievalOwnerFilter } from "@/lib/owner-scope"; import { buildDocumentIndexUnitInputs, countDocumentIndexUnitsByType, @@ -823,11 +823,11 @@ export async function fetchMemoryCardsForQuery(args: { match_count: args.matchCount ?? 32, min_similarity: 0.1, document_filters: args.documentIds?.length ? args.documentIds : null, - owner_filter: args.ownerId - ? requireOwnerScope(args.ownerId) - : args.documentIds?.length - ? null - : (requireOwnerScope(args.ownerId) ?? null), + owner_filter: retrievalOwnerFilter({ + ownerId: args.ownerId, + documentIds: args.documentIds, + allowGlobalSearch: !args.ownerId && !args.documentIds?.length, + }), }); if (error) { diff --git a/src/lib/document-enrichment.ts b/src/lib/document-enrichment.ts index 6aee1d8d0..ec77e4670 100644 --- a/src/lib/document-enrichment.ts +++ b/src/lib/document-enrichment.ts @@ -1,7 +1,7 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { classifyDocumentOrganization } from "@/lib/document-organization"; import { env } from "@/lib/env"; -import { requireOwnerScope } from "@/lib/owner-scope"; +import { retrievalOwnerFilter } from "@/lib/owner-scope"; import { isClinicalImageEvidence } from "@/lib/image-filtering"; import { buildCoveragePromptNote, @@ -713,7 +713,7 @@ export async function fetchRelatedDocumentMetadata(args: { }) { const { data: rpcData, error: rpcError } = await args.supabase.rpc("get_related_document_metadata", { document_ids: args.documentIds, - owner_filter: args.ownerId ? requireOwnerScope(args.ownerId) : null, + owner_filter: retrievalOwnerFilter({ ownerId: args.ownerId, documentIds: args.documentIds }), }); if (!rpcError) { diff --git a/src/lib/owner-scope.ts b/src/lib/owner-scope.ts index 4bec91eca..4eec27540 100644 --- a/src/lib/owner-scope.ts +++ b/src/lib/owner-scope.ts @@ -1,17 +1,7 @@ import { isDemoMode, isLocalNoAuthMode } from "@/lib/env"; -/** - * Fail-closed guard for multi-tenant owner scoping. - * - * The hybrid retrieval RPCs treat a null `owner_filter` as "all owners" - * (fail-open), so calling them without an ownerId would silently return another - * tenant's data. Call this at every retrieval RPC boundary that filters by - * owner. In a real (multi-user) deployment a missing ownerId throws instead of - * leaking; in demo / local-no-auth / the test runner there is no multi-tenancy, - * so it stays permissive (returns undefined, preserving the previous behaviour). - * - * See the owner-scoping isolation audit. - */ +export const PUBLIC_OWNER_FILTER_SENTINEL = "00000000-0000-0000-0000-000000000000"; + export function requireOwnerScope(ownerId: string | null | undefined): string | undefined { if (ownerId) return ownerId; if (isDemoMode() || isLocalNoAuthMode() || process.env.NODE_ENV === "test") { @@ -21,3 +11,20 @@ export function requireOwnerScope(ownerId: string | null | undefined): string | "Owner-scoped retrieval was called without an ownerId; refusing to run to avoid returning another tenant's data.", ); } + +export function retrievalOwnerFilter(args: { + ownerId?: string | null; + documentIds?: string[]; + allowGlobalSearch?: boolean; +}): string | null | undefined { + if (args.ownerId) return requireOwnerScope(args.ownerId); + if (isDemoMode() || isLocalNoAuthMode() || process.env.NODE_ENV === "test") { + return undefined; + } + if (args.allowGlobalSearch || args.documentIds?.length) { + return PUBLIC_OWNER_FILTER_SENTINEL; + } + throw new Error( + "Owner-scoped retrieval was called without an ownerId; refusing to run to avoid returning another tenant's data.", + ); +} diff --git a/src/lib/rag.ts b/src/lib/rag.ts index e43a125e6..e0afa8b45 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1,5 +1,5 @@ import { createAdminClient } from "@/lib/supabase/admin"; -import { requireOwnerScope } from "@/lib/owner-scope"; +import { requireOwnerScope, retrievalOwnerFilter } from "@/lib/owner-scope"; import type { Database, Json } from "@/lib/supabase/database.types"; import { embedTextWithTelemetry, @@ -2062,10 +2062,12 @@ function assertGlobalSearchAllowed(args: SearchChunksArgs) { } } -function ownerScopeForDocumentFilteredRetrieval(ownerId: string | undefined, documentIds: string[] | undefined) { - if (ownerId) return requireOwnerScope(ownerId); - if (documentIds?.length) return undefined; - return requireOwnerScope(ownerId); +function ownerScopeForDocumentFilteredRetrieval( + ownerId: string | undefined, + documentIds: string[] | undefined, + allowGlobalSearch?: boolean, +) { + return retrievalOwnerFilter({ ownerId, documentIds, allowGlobalSearch }); } export function buildRetrievalQueryVariants( @@ -2193,6 +2195,7 @@ async function searchTextChunkCandidates(args: { queryVariants: string[]; ownerId?: string; documentIds?: string[]; + allowGlobalSearch?: boolean; matchCount: number; }) { const runChunkText = async (queryText: string, matchCount: number) => { @@ -2200,7 +2203,7 @@ async function searchTextChunkCandidates(args: { query_text: queryText, match_count: matchCount, document_filters: args.documentIds ?? undefined, - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); return error || !data?.length ? ([] as SearchResult[]) : (data as SearchResult[]); }; @@ -2347,13 +2350,14 @@ async function fetchBestDocumentLookupChunks(args: { query: string; limit: number; ownerId?: string; + allowGlobalSearch?: boolean; }) { const terms = documentLookupChunkTerms(args.query); const { data: rpcChunks, error: rpcError } = await args.supabase.rpc("match_document_lookup_chunks_text", { query_text: args.query, document_filters: args.documentIds ?? undefined, match_count: Math.max(args.limit * 3, 24), - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); if (!rpcError && rpcChunks?.length) { const ranked = (rpcChunks as DocumentLookupChunkRow[]) @@ -2741,6 +2745,7 @@ async function loadChunksForSignalMatches(args: { .in("id", documentIds) .eq("status", "indexed"); if (args.ownerId) documentQuery = documentQuery.eq("owner_id", args.ownerId); + else documentQuery = documentQuery.is("owner_id", null); const { data: documents, error: documentsError } = await documentQuery; if (documentsError || !documents?.length) return [] as SearchResult[]; const documentById = new Map(documents.map((document) => [document.id, document])); @@ -2800,6 +2805,7 @@ async function searchTableFactCandidates(args: { queryVariants?: string[]; ownerId?: string; documentIds?: string[]; + allowGlobalSearch?: boolean; matchCount: number; }) { const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( @@ -2812,7 +2818,7 @@ async function searchTableFactCandidates(args: { query_text: variant, match_count: index === 0 ? args.matchCount : Math.min(args.matchCount, 24), document_filters: args.documentIds ?? undefined, - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); if (error || !data?.length) return [] as TableFactRpcRow[]; return data as TableFactRpcRow[]; @@ -2851,6 +2857,7 @@ async function searchEmbeddingFieldCandidates(args: { queryEmbedding: number[]; ownerId?: string; documentIds?: string[]; + allowGlobalSearch?: boolean; matchCount: number; telemetry?: SearchTelemetry; }) { @@ -2860,7 +2867,7 @@ async function searchEmbeddingFieldCandidates(args: { match_count: args.matchCount, min_similarity: 0.12, document_filters: args.documentIds ?? undefined, - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); if (error) recordHybridRpcError(args.telemetry, "match_document_embedding_fields_hybrid", error); if (error || !data?.length) return [] as SearchResult[]; @@ -2901,6 +2908,7 @@ async function searchIndexUnitCandidates(args: { queryEmbedding: number[]; ownerId?: string; documentIds?: string[]; + allowGlobalSearch?: boolean; matchCount: number; telemetry?: SearchTelemetry; }) { @@ -2910,7 +2918,7 @@ async function searchIndexUnitCandidates(args: { match_count: args.matchCount, min_similarity: 0.1, document_filters: args.documentIds ?? undefined, - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); if (error) recordHybridRpcError(args.telemetry, "match_document_index_units_hybrid", error); if (error || !data?.length) return [] as SearchResult[]; @@ -5525,6 +5533,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { queryVariants, ownerId: args.ownerId, documentIds: documentFilterList, + allowGlobalSearch: args.allowGlobalSearch, matchCount: textCandidateCount, }); telemetry.text_candidate_count = textData.length; @@ -5623,6 +5632,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { queryVariants, ownerId: args.ownerId, documentIds: documentFilterList, + allowGlobalSearch: args.allowGlobalSearch, matchCount: Math.min(candidateCount, 48), }); const tableFactLatencyMs = Date.now() - tableFactStartedAt; @@ -5803,6 +5813,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { queryEmbedding: embedding, ownerId: args.ownerId, documentIds: documentFilterList, + allowGlobalSearch: args.allowGlobalSearch, matchCount: Math.min(candidateCount, 48), telemetry, }); @@ -5816,6 +5827,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { queryEmbedding: embedding, ownerId: args.ownerId, documentIds: documentFilterList, + allowGlobalSearch: args.allowGlobalSearch, matchCount: Math.min(candidateCount, 64), telemetry, }); @@ -5829,7 +5841,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { match_count: candidateCount, min_similarity: minSimilarity, document_filters: documentFilterList ?? undefined, - owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, documentFilterList), + owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, documentFilterList, args.allowGlobalSearch), }); return { data, error, latencyMs: Date.now() - startedAt }; })(), @@ -5928,6 +5940,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { owner_filter: ownerScopeForDocumentFilteredRetrieval( args.ownerId, documentFilter ? [documentFilter] : undefined, + documentFilter ? undefined : args.allowGlobalSearch, ), }); diff --git a/supabase/migrations/20260705210000_retrieval_owner_filter_sentinel.sql b/supabase/migrations/20260705210000_retrieval_owner_filter_sentinel.sql new file mode 100644 index 000000000..9a973f440 --- /dev/null +++ b/supabase/migrations/20260705210000_retrieval_owner_filter_sentinel.sql @@ -0,0 +1,1030 @@ +-- Public-only retrieval owner filter sentinel for hybrid RPCs. +set search_path = public, extensions, pg_temp; + +create or replace function public.retrieval_owner_matches(owner_filter uuid, row_owner_id uuid) +returns boolean +language sql +immutable +parallel safe +set search_path = public, pg_temp +as $$ + select case + when owner_filter is null then true + when owner_filter = '00000000-0000-0000-0000-000000000000'::uuid then row_owner_id is null + else row_owner_id = owner_filter + end; +$$; + +create or replace function public.match_document_chunks( + query_embedding extensions.vector(1536), + match_count integer default 8, + min_similarity double precision default 0.15, + document_filter uuid default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + '[]'::jsonb as document_labels, + null::text as document_summary, + 1 - (c.embedding <=> query_embedding) as similarity, + '[]'::jsonb as images + from public.document_chunks c + join public.documents d on d.id = c.document_id + where (document_filter is null or c.document_id = document_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit match_count; +$$; + +create or replace function public.match_document_chunks( + query_embedding extensions.vector(1536), + match_count integer default 8, + min_similarity double precision default 0.15, + document_filter uuid default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + '[]'::jsonb as document_labels, + null::text as document_summary, + 1 - (c.embedding <=> query_embedding) as similarity, + '[]'::jsonb as images + from public.document_chunks c + join public.documents d on d.id = c.document_id + where (document_filter is null or c.document_id = document_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit match_count; +$$; + +create or replace function public.match_document_chunks_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 12, + min_similarity double precision default 0.12, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + row_number() over (order by c.embedding <=> query_embedding) as vector_rank, + null::bigint as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and 1 - (c.embedding <=> query_embedding) >= min_similarity + order by c.embedding <=> query_embedding + limit greatest(match_count * 6, 48) + ), + text_ranked as ( + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + 1 - (c.embedding <=> query_embedding) as similarity, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank, + null::bigint as vector_rank, + row_number() over ( + order by + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc, + c.embedding <=> query_embedding + ) as text_match_rank, + coalesce((d.metadata->'rag_indexing_version') is not null, false) as has_deep_index, + d.updated_at as doc_updated_at, + coalesce( + (select q.quality_score from public.document_index_quality q where q.document_id = c.document_id), + 0.7 + ) as quality_score + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and c.search_tsv @@ query.tsq + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit greatest(match_count * 6, 48) + ), + combined as ( + select * from vector_ranked + union all + select * from text_ranked + ), + scored as ( + select + id, + document_id, + page_number, + chunk_index, + section_heading, + content, + retrieval_synopsis, + image_ids, + max(similarity)::double precision as similarity, + max(text_rank)::double precision as text_rank, + min(vector_rank) as vector_rank, + min(text_match_rank) as text_match_rank, + max(quality_score)::double precision as quality_score, + bool_or(has_deep_index) as has_deep_index, + max(doc_updated_at) as doc_updated_at + from combined + group by id, document_id, page_number, chunk_index, section_heading, content, retrieval_synopsis, image_ids + ), + scored_metrics as ( + select + scored.*, + ( + (scored.similarity * 0.62) + + (least(scored.text_rank, 1) * 0.22) + + (scored.quality_score * 0.10) + + (case when scored.doc_updated_at > now() - interval '90 days' then 0.06 else 0 end) + )::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + scored.vector_rank), 0) + + coalesce(1.0 / (60 + scored.text_match_rank), 0) + )::double precision as rrf_score + from scored + ), + hybrid_candidates as ( + select id + from scored_metrics + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count + ), + vector_candidates as ( + select id + from scored_metrics + order by similarity desc, hybrid_score desc + limit match_count + ), + text_candidates as ( + select id + from scored_metrics + order by text_rank desc, hybrid_score desc + limit match_count + ), + rrf_candidates as ( + select id + from scored_metrics + order by rrf_score desc, hybrid_score desc + limit match_count + ), + candidate_ids as ( + select id from hybrid_candidates + union + select id from vector_candidates + union + select id from text_candidates + union + select id from rrf_candidates + ) + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + c.similarity, + c.text_rank, + c.hybrid_score, + c.rrf_score, + public.chunk_image_metadata(c.image_ids) as images + from scored_metrics c + join candidate_ids candidates on candidates.id = c.id + join public.documents d on d.id = c.document_id + order by c.hybrid_score desc, c.rrf_score desc, c.similarity desc, c.text_rank desc + limit match_count; +$$; + +create or replace function public.match_document_memory_cards_hybrid_v2( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 32, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + owner_id uuid, + section_id uuid, + card_type text, + title text, + content text, + normalized_terms text[], + page_number integer, + source_chunk_ids uuid[], + source_image_ids uuid[], + confidence real, + metadata jsonb, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_ranked as ( + select + m.*, + (1 - (m.embedding <=> query_embedding))::double precision as similarity, + ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank, + row_number() over (order by m.embedding <=> query_embedding) as vector_rank, + null::bigint as text_match_rank + from public.document_memory_cards m + join public.documents d on d.id = m.document_id + cross join query + where (document_filters is null or m.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_artifact_generation(m.metadata, d.metadata) + and (1 - (m.embedding <=> query_embedding)) >= min_similarity + order by m.embedding <=> query_embedding + limit greatest(match_count * 6, 96) + ), + text_ranked as ( + select + m.*, + (1 - (m.embedding <=> query_embedding))::double precision as similarity, + ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank, + null::bigint as vector_rank, + row_number() over ( + order by ts_rank_cd(m.search_tsv, query.tsq) desc, m.embedding <=> query_embedding + ) as text_match_rank + from public.document_memory_cards m + join public.documents d on d.id = m.document_id + cross join query + where (document_filters is null or m.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_artifact_generation(m.metadata, d.metadata) + and m.search_tsv @@ query.tsq + order by ts_rank_cd(m.search_tsv, query.tsq) desc + limit greatest(match_count * 6, 96) + ), + combined as ( + select * from vector_ranked + union all + select * from text_ranked + ), + scored as ( + select + id, document_id, owner_id, section_id, card_type, title, content, normalized_terms, + page_number, source_chunk_ids, source_image_ids, confidence, metadata, + max(similarity)::double precision as similarity, + max(text_rank)::double precision as text_rank, + min(vector_rank) as vector_rank, + min(text_match_rank) as text_match_rank + from combined + group by + id, document_id, owner_id, section_id, card_type, title, content, normalized_terms, + page_number, source_chunk_ids, source_image_ids, confidence, metadata + ) + select + id, document_id, owner_id, section_id, card_type, title, content, normalized_terms, + page_number, source_chunk_ids, source_image_ids, confidence, metadata, similarity, text_rank, + ( + (similarity * 0.62) + + (least(text_rank, 1) * 0.24) + + (confidence * 0.10) + + ( + coalesce(1.0 / (60 + vector_rank), 0) + + coalesce(1.0 / (60 + text_match_rank), 0) + ) * 0.04 + )::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + vector_rank), 0) + + coalesce(1.0 / (60 + text_match_rank), 0) + )::double precision as rrf_score + from scored + order by hybrid_score desc, similarity desc, text_rank desc, confidence desc + limit match_count; +$$; + +create or replace function public.match_documents_for_query( + query_text text, + match_count integer default 12, + owner_filter uuid default null +) +returns table ( + id uuid, + owner_id uuid, + title text, + file_name text, + status text, + page_count integer, + chunk_count integer, + image_count integer, + metadata jsonb, + text_rank double precision, + match_reason text +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select + websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + lower(coalesce(query_text, '')) as normalized + ), + ranked as ( + select + d.id, + d.owner_id, + d.title, + d.file_name, + d.status, + d.page_count, + d.chunk_count, + d.image_count, + d.metadata, + ( + (ts_rank_cd(d.title_search_tsv, query.tsq) * 4.0) + + (ts_rank_cd(d.search_tsv, query.tsq) * 1.5) + + coalesce(max(ts_rank_cd(to_tsvector('english', l.label), query.tsq)) * 1.2, 0) + + coalesce(ts_rank_cd(to_tsvector('english', s.summary), query.tsq), 0) + + (greatest( + similarity(lower(coalesce(d.title, '') || ' ' || coalesce(d.file_name, '')), query.normalized), + coalesce(max(similarity(lower(l.label), query.normalized)), 0), + coalesce(similarity(lower(s.summary), query.normalized), 0) + ) * 1.6) + )::double precision as text_rank, + case + when d.title_search_tsv @@ query.tsq then 'title' + when max(l.label) filter (where to_tsvector('english', l.label) @@ query.tsq) is not null then 'label' + when s.summary is not null and to_tsvector('english', s.summary) @@ query.tsq then 'summary' + when similarity(lower(coalesce(d.title, '') || ' ' || coalesce(d.file_name, '')), query.normalized) >= 0.18 then 'fuzzy_title' + when d.search_tsv @@ query.tsq then 'metadata' + else 'none' + end as match_reason + from public.documents d + left join public.document_labels l + on l.document_id = d.id + and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' + and coalesce(l.metadata->>'hidden', 'false') <> 'true' + left join public.document_summaries s on s.document_id = d.id + cross join query + where public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and ( + d.title_search_tsv @@ query.tsq + or d.search_tsv @@ query.tsq + or to_tsvector('english', coalesce(l.label, '')) @@ query.tsq + or to_tsvector('english', coalesce(s.summary, '')) @@ query.tsq + or similarity(lower(coalesce(d.title, '') || ' ' || coalesce(d.file_name, '')), query.normalized) >= 0.18 + or similarity(lower(coalesce(l.label, '')), query.normalized) >= 0.2 + or similarity(lower(coalesce(s.summary, '')), query.normalized) >= 0.16 + ) + group by d.id, d.owner_id, d.title, d.file_name, d.status, d.page_count, d.chunk_count, d.image_count, d.metadata, s.summary, query.tsq, query.normalized + ) + select * + from ranked + where text_rank > 0 + order by text_rank desc, page_count desc, title asc + limit match_count; +$$; + +create or replace function public.match_document_chunks_text( + query_text text, + match_count integer default 12, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + lexical_score double precision, + images jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + ranked as ( + select + c.id, + c.document_id, + d.title, + d.file_name, + c.page_number, + c.chunk_index, + c.section_heading, + c.content, + c.retrieval_synopsis, + c.image_ids, + d.metadata as source_metadata, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + )::double precision as text_rank + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where (document_filters is null or c.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) + order by ( + ts_rank_cd(c.search_tsv, query.tsq) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0) + ) desc + limit least(greatest(match_count * 2, 24), 96) + ), + -- Batch-fetch label metadata for all distinct document_ids in the result set. + -- One query replaces N per-row calls to document_label_metadata(). + doc_labels as ( + select + l.document_id, + coalesce( + jsonb_agg( + jsonb_build_object( + 'id', l.id, + 'document_id', l.document_id, + 'owner_id', l.owner_id, + 'label', l.label, + 'label_type', l.label_type, + 'source', l.source, + 'confidence', l.confidence, + 'metadata', l.metadata, + 'created_at', l.created_at, + 'updated_at', l.updated_at + ) + order by l.confidence desc, l.label + ), + '[]'::jsonb + ) as labels + from public.document_labels l + where l.document_id in (select distinct ranked.document_id from ranked) + and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' + and coalesce(l.metadata->>'hidden', 'false') <> 'true' + group by l.document_id + ), + -- Batch-fetch summary text for all distinct document_ids in the result set. + -- One query replaces N per-row calls to document_summary_text(). + doc_summaries as ( + select distinct on (s.document_id) + s.document_id, + s.summary + from public.document_summaries s + where s.document_id in (select distinct ranked.document_id from ranked) + order by s.document_id + ) + select + ranked.id, + ranked.document_id, + ranked.title, + ranked.file_name, + ranked.page_number, + ranked.chunk_index, + ranked.section_heading, + ranked.content, + ranked.retrieval_synopsis, + ranked.image_ids, + ranked.source_metadata, + coalesce(doc_labels.labels, '[]'::jsonb) as document_labels, + doc_summaries.summary as document_summary, + -- Text-only fallback has NO vector cosine similarity. Do not fabricate one: + -- a synthetic value here was read downstream as a real semantic score and + -- could label a pure keyword hit as "strong"/"moderate" evidence (>=0.64). + -- Leave similarity at 0; the lexical signal lives in lexical_score. + 0::double precision as similarity, + ranked.text_rank, + -- Cap hybrid_score well below the 0.64 "moderate" threshold so a lexical-only + -- row can order amongst its peers but can never masquerade as a moderate/strong + -- cosine match when merged with vector results. + least(0.5, 0.18 + (least(ranked.text_rank, 1) * 0.3))::double precision as hybrid_score, + least(0.99, 0.4 + (least(ranked.text_rank, 1) * 0.59))::double precision as lexical_score, + public.chunk_image_metadata(ranked.image_ids) as images + from ranked + left join doc_labels on doc_labels.document_id = ranked.document_id + left join doc_summaries on doc_summaries.document_id = ranked.document_id + order by lexical_score desc, text_rank desc + limit match_count; +$$; + +create or replace function public.match_document_lookup_chunks_text( + query_text text, + document_filters uuid[], + match_count integer default 24, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + page_number integer, + chunk_index integer, + section_heading text, + section_path text[], + heading_level integer, + parent_heading text, + anchor_id text, + content text, + retrieval_synopsis text, + image_ids uuid[], + text_rank double precision +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ) + select + c.id, + c.document_id, + c.page_number, + c.chunk_index, + c.section_heading, + c.section_path, + c.heading_level, + c.parent_heading, + c.anchor_id, + c.content, + c.retrieval_synopsis, + c.image_ids, + ( + ts_rank_cd(c.search_tsv, query.tsq) + + (case when c.section_heading is not null then ts_rank_cd(to_tsvector('english', c.section_heading), query.tsq) * 0.35 else 0 end) + + (ts_rank_cd(d.title_search_tsv, query.tsq) * 0.25) + )::double precision as text_rank + from public.document_chunks c + join public.documents d on d.id = c.document_id + cross join query + where document_filters is not null + and c.document_id = any(document_filters) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_document_generation(c.index_generation_id, d.metadata) + and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) + order by text_rank desc, c.chunk_index asc + limit least(greatest(match_count, 1), 80); +$$; + +create or replace function public.get_related_document_metadata( + document_ids uuid[], + owner_filter uuid default null +) +returns table ( + document_id uuid, + labels jsonb, + summary text +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + select + d.id as document_id, + coalesce( + ( + select jsonb_agg( + jsonb_build_object( + 'id', l.id, + 'document_id', l.document_id, + 'owner_id', l.owner_id, + 'label', l.label, + 'label_type', l.label_type, + 'source', l.source, + 'confidence', l.confidence, + 'metadata', l.metadata, + 'created_at', l.created_at, + 'updated_at', l.updated_at + ) + order by l.confidence desc, l.label + ) + from public.document_labels l + where l.document_id = d.id + and public.retrieval_owner_matches(owner_filter, l.owner_id) + and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' + and coalesce(l.metadata->>'hidden', 'false') <> 'true' + ), + '[]'::jsonb + ) as labels, + ( + select s.summary + from public.document_summaries s + where s.document_id = d.id + and public.retrieval_owner_matches(owner_filter, s.owner_id) + order by s.generated_at desc + limit 1 + ) as summary + from public.documents d + where d.id = any(document_ids) + and public.retrieval_owner_matches(owner_filter, d.owner_id); +$$; + +create or replace function public.match_document_table_facts_text( + query_text text, + match_count integer default 16, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + page_number integer, + table_title text, + row_label text, + clinical_parameter text, + threshold_value text, + action text, + text_rank double precision, + match_reason text, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select + websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + lower(coalesce(query_text, '')) as normalized, + regexp_split_to_array(lower(coalesce(query_text, '')), '[^a-z0-9]+') as terms + ), + ranked as ( + select + f.id, + f.document_id, + f.source_chunk_id, + f.source_image_id, + f.page_number, + f.table_title, + f.row_label, + f.clinical_parameter, + f.threshold_value, + f.action, + ( + ts_rank_cd(f.search_tsv, query.tsq) + + ( + similarity( + lower( + coalesce(f.table_title, '') || ' ' || + coalesce(f.row_label, '') || ' ' || + coalesce(f.clinical_parameter, '') || ' ' || + coalesce(f.threshold_value, '') || ' ' || + coalesce(f.action, '') + ), + query.normalized + ) * 0.8 + ) + + case + when coalesce(f.threshold_value, '') <> '' + and regexp_split_to_array(lower(f.threshold_value), '[^a-z0-9]+') && query.terms then 0.12 + else 0 + end + + case + when coalesce(f.action, '') <> '' + and regexp_split_to_array(lower(f.action), '[^a-z0-9]+') && query.terms then 0.1 + else 0 + end + )::double precision as text_rank, + case + when coalesce(f.threshold_value, '') <> '' then 'table_threshold' + when coalesce(f.action, '') <> '' then 'table_action' + else 'table_row' + end as match_reason, + f.metadata + from public.document_table_facts f + join public.documents d on d.id = f.document_id + cross join query + where (document_filters is null or f.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, f.owner_id) + and d.status = 'indexed' + and public.is_committed_artifact_generation(f.metadata, d.metadata) + and ( + f.search_tsv @@ query.tsq + or f.normalized_terms && query.terms + or similarity( + lower( + coalesce(f.table_title, '') || ' ' || + coalesce(f.row_label, '') || ' ' || + coalesce(f.clinical_parameter, '') || ' ' || + coalesce(f.threshold_value, '') || ' ' || + coalesce(f.action, '') + ), + query.normalized + ) >= 0.18 + ) + ) + select * + from ranked + where text_rank > 0 + order by text_rank desc, page_number asc nulls last + limit match_count; +$$; + +create or replace function public.match_document_embedding_fields_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 16, + min_similarity double precision default 0.5, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + field_type text, + content text, + similarity double precision, + text_rank double precision, + hybrid_score double precision +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq + ), + vector_hits as ( + select f.id + from public.document_embedding_fields f + join public.documents d on d.id = f.document_id + where (document_filters is null or f.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_artifact_generation(f.metadata, d.metadata) + and f.source_chunk_id is not null + and 1 - (f.embedding <=> query_embedding) >= min_similarity + order by f.embedding <=> query_embedding + limit greatest(match_count * 3, 32) + ), + text_hits as ( + select f.id + from public.document_embedding_fields f + join public.documents d on d.id = f.document_id + cross join query + where (document_filters is null or f.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and d.status = 'indexed' + and public.is_committed_artifact_generation(f.metadata, d.metadata) + and f.source_chunk_id is not null + and f.search_tsv @@ query.tsq + order by ts_rank_cd(f.search_tsv, query.tsq) desc + limit greatest(match_count * 3, 32) + ), + candidate_ids as ( + select id from vector_hits + union + select id from text_hits + ), + ranked as ( + select + f.id, f.document_id, f.source_chunk_id, f.field_type, f.content, + (1 - (f.embedding <=> query_embedding))::double precision as similarity, + ts_rank_cd(f.search_tsv, query.tsq)::double precision as text_rank + from public.document_embedding_fields f + join candidate_ids ci on ci.id = f.id + cross join query + ) + select + id, document_id, source_chunk_id, field_type, content, similarity, text_rank, + ((similarity * 0.7) + (least(text_rank, 1) * 0.3))::double precision as hybrid_score + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; + +create or replace function public.match_document_index_units_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + ranked as ( + select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, + u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact', 'threshold', 'workflow_step', 'medication_monitoring', 'alias', 'visual_summary', 'flowchart_step', 'diagram_decision', 'risk_matrix_cell', 'medication_chart_row', 'chart_finding', 'visual_askable_question', 'table_threshold') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, + u.metadata + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and public.retrieval_owner_matches(owner_filter, d.owner_id) + and public.is_committed_artifact_generation(u.metadata, d.metadata) + and u.source_chunk_id is not null + and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) + order by text_rank desc + limit greatest(match_count * 3, 48) + ) + select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, + normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type in ('askable_question', 'threshold', 'table_fact', 'table_threshold', 'visual_askable_question') then 0.04 + when unit_type in ('workflow_step', 'medication_monitoring', 'flowchart_step', 'diagram_decision', 'medication_chart_row', 'risk_matrix_cell') then 0.03 + else 0 end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; diff --git a/supabase/migrations/20260705220000_promote_locally_reviewed_documents_public.sql b/supabase/migrations/20260705220000_promote_locally_reviewed_documents_public.sql index 76c9a5560..f6439c7f1 100644 --- a/supabase/migrations/20260705220000_promote_locally_reviewed_documents_public.sql +++ b/supabase/migrations/20260705220000_promote_locally_reviewed_documents_public.sql @@ -43,12 +43,12 @@ from promoted_public_documents pd where dmc.document_id = pd.id; update public.document_table_facts dtf -set owner_id = null, updated_at = now() +set owner_id = null from promoted_public_documents pd where dtf.document_id = pd.id; update public.document_embedding_fields def -set owner_id = null, updated_at = now() +set owner_id = null from promoted_public_documents pd where def.document_id = pd.id; diff --git a/supabase/schema.sql b/supabase/schema.sql index 16ce2230c..357ed9180 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -1903,6 +1903,20 @@ as $$ nullif(coalesce(document_metadata, '{}'::jsonb)->>'index_generation_id', ''); $$; +create or replace function public.retrieval_owner_matches(owner_filter uuid, row_owner_id uuid) +returns boolean +language sql +immutable +parallel safe +set search_path = public, pg_temp +as $$ + select case + when owner_filter is null then true + when owner_filter = '00000000-0000-0000-0000-000000000000'::uuid then row_owner_id is null + else row_owner_id = owner_filter + end; +$$; + create or replace function public.match_document_chunks( query_embedding extensions.vector(1536), match_count integer default 8, @@ -1950,7 +1964,7 @@ as $$ from public.document_chunks c join public.documents d on d.id = c.document_id where (document_filter is null or c.document_id = document_filter) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_document_generation(c.index_generation_id, d.metadata) and 1 - (c.embedding <=> query_embedding) >= min_similarity @@ -2018,7 +2032,7 @@ as $$ join public.documents d on d.id = c.document_id cross join query where (document_filters is null or c.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_document_generation(c.index_generation_id, d.metadata) and 1 - (c.embedding <=> query_embedding) >= min_similarity @@ -2059,7 +2073,7 @@ as $$ join public.documents d on d.id = c.document_id cross join query where (document_filters is null or c.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_document_generation(c.index_generation_id, d.metadata) and c.search_tsv @@ query.tsq @@ -2211,7 +2225,7 @@ as $$ join public.documents d on d.id = m.document_id cross join query where (document_filters is null or m.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_artifact_generation(m.metadata, d.metadata) and (1 - (m.embedding <=> query_embedding)) >= min_similarity @@ -2231,7 +2245,7 @@ as $$ join public.documents d on d.id = m.document_id cross join query where (document_filters is null or m.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_artifact_generation(m.metadata, d.metadata) and m.search_tsv @@ query.tsq @@ -2629,7 +2643,7 @@ as $$ and coalesce(l.metadata->>'hidden', 'false') <> 'true' left join public.document_summaries s on s.document_id = d.id cross join query - where (owner_filter is null or d.owner_id = owner_filter) + where public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and ( d.title_search_tsv @@ query.tsq @@ -2703,7 +2717,7 @@ as $$ join public.documents d on d.id = c.document_id cross join query where (document_filters is null or c.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_document_generation(c.index_generation_id, d.metadata) and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) @@ -2836,7 +2850,7 @@ as $$ cross join query where document_filters is not null and c.document_id = any(document_filters) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_document_generation(c.index_generation_id, d.metadata) and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq) @@ -2881,7 +2895,7 @@ as $$ ) from public.document_labels l where l.document_id = d.id - and (owner_filter is null or l.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, l.owner_id) and coalesce(l.metadata->>'review_status', 'new') <> 'hidden' and coalesce(l.metadata->>'hidden', 'false') <> 'true' ), @@ -2891,13 +2905,13 @@ as $$ select s.summary from public.document_summaries s where s.document_id = d.id - and (owner_filter is null or s.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, s.owner_id) order by s.generated_at desc limit 1 ) as summary from public.documents d where d.id = any(document_ids) - and (owner_filter is null or d.owner_id = owner_filter); + and public.retrieval_owner_matches(owner_filter, d.owner_id); $$; create or replace function public.match_document_table_facts_text( @@ -2978,7 +2992,7 @@ as $$ join public.documents d on d.id = f.document_id cross join query where (document_filters is null or f.document_id = any(document_filters)) - and (owner_filter is null or f.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, f.owner_id) and d.status = 'indexed' and public.is_committed_artifact_generation(f.metadata, d.metadata) and ( @@ -3033,7 +3047,7 @@ as $$ from public.document_embedding_fields f join public.documents d on d.id = f.document_id where (document_filters is null or f.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_artifact_generation(f.metadata, d.metadata) and f.source_chunk_id is not null @@ -3047,7 +3061,7 @@ as $$ join public.documents d on d.id = f.document_id cross join query where (document_filters is null or f.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and d.status = 'indexed' and public.is_committed_artifact_generation(f.metadata, d.metadata) and f.source_chunk_id is not null @@ -3938,7 +3952,7 @@ as $$ cross join query where d.status = 'indexed' and (document_filters is null or u.document_id = any(document_filters)) - and (owner_filter is null or d.owner_id = owner_filter) + and public.retrieval_owner_matches(owner_filter, d.owner_id) and public.is_committed_artifact_generation(u.metadata, d.metadata) and u.source_chunk_id is not null and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) diff --git a/tests/owner-scope.test.ts b/tests/owner-scope.test.ts index a47a6bbdc..33030b816 100644 --- a/tests/owner-scope.test.ts +++ b/tests/owner-scope.test.ts @@ -27,3 +27,12 @@ describe("requireOwnerScope (fail-closed owner scoping)", () => { expect(() => requireOwnerScope(null)).toThrow(/tenant/); }); }); + +describe("retrievalOwnerFilter", () => { + it("returns the public sentinel for anonymous production global search", async () => { + vi.doMock("@/lib/env", () => ({ isDemoMode: () => false, isLocalNoAuthMode: () => false })); + vi.stubEnv("NODE_ENV", "production"); + const { retrievalOwnerFilter, PUBLIC_OWNER_FILTER_SENTINEL } = await import("../src/lib/owner-scope"); + expect(retrievalOwnerFilter({ allowGlobalSearch: true })).toBe(PUBLIC_OWNER_FILTER_SENTINEL); + }); +}); diff --git a/tests/public-access-deep.test.ts b/tests/public-access-deep.test.ts index 9d0cf39d1..a5ad9ea60 100644 --- a/tests/public-access-deep.test.ts +++ b/tests/public-access-deep.test.ts @@ -124,7 +124,7 @@ describe("production anonymous retrieval scope", () => { vi.unstubAllEnvs(); }); - it("fails closed in production when anonymous global retrieval omits owner scope", async () => { + it("rejects anonymous global retrieval without allowGlobalSearch in production", async () => { vi.doUnmock("@/lib/rag"); vi.stubEnv("NODE_ENV", "production"); vi.doMock("@/lib/env", () => ({ @@ -160,10 +160,50 @@ describe("production anonymous retrieval scope", () => { await expect( searchChunksWithTelemetry({ query: "clozapine monitoring", - allowGlobalSearch: true, }), ).rejects.toThrow(/ownerId|tenant/i); }); + + it("scopes anonymous global retrieval to public documents when allowGlobalSearch is true", async () => { + vi.doUnmock("@/lib/rag"); + vi.stubEnv("NODE_ENV", "production"); + vi.doMock("@/lib/env", () => ({ + env: { + OPENAI_API_KEY: "sk-test", + RAG_PROVIDER_MODE: "offline", + RAG_SEARCH_CACHE_TTL_MS: 0, + RAG_ANSWER_CACHE_TTL_MS: 0, + }, + isDemoMode: () => false, + isLocalNoAuthMode: () => false, + requestedOpenAIAnswerModels: () => ({ fast: "gpt-test", strong: "gpt-test" }), + })); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: vi.fn(() => ({ + from: vi.fn(() => ({ + select: vi.fn(() => ({ + eq: vi.fn(() => ({ + is: vi.fn(() => ({ + order: vi.fn(() => ({ + limit: vi.fn(async () => ({ data: [], error: null })), + })), + })), + })), + })), + })), + rpc: vi.fn(async () => ({ data: [], error: null })), + })), + })); + + const { searchChunksWithTelemetry } = await import("../src/lib/rag"); + const result = await searchChunksWithTelemetry({ + query: "clozapine monitoring", + allowGlobalSearch: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry).toBeDefined(); + }); }); describe("test-runtime anonymous retrieval scope", () => { diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 6dd16d6c9..5aeb3ce89 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -520,7 +520,13 @@ describe("Supabase schema Data API grants", () => { it("filters hybrid retrieval by owner inside Postgres", () => { expect(schema).toContain("owner_filter uuid default null"); - expect(schema).toContain("and (owner_filter is null or d.owner_id = owner_filter)"); + expect(schema).toContain( + "create or replace function public.retrieval_owner_matches(owner_filter uuid, row_owner_id uuid)", + ); + expect(schema).toContain( + "when owner_filter = '00000000-0000-0000-0000-000000000000'::uuid then row_owner_id is null", + ); + expect(schema).toContain("and public.retrieval_owner_matches(owner_filter, d.owner_id)"); expect(schema).toContain("create or replace function public.match_document_chunks_text"); expect(schema).toContain("create or replace function public.match_document_chunks_hybrid"); expect(schema).toContain("rrf_score double precision");