From 2cdcda3230de50f2d62bdf9485d35117e20b038d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:59:34 +0800 Subject: [PATCH] refactor(rag): extract candidate-source searchers into rag-candidate-sources.ts Audit 2026-07-13 deferred item D3, rag.ts decomposition phase 1 (finding 11). Zero behavior change: 1,014 lines moved verbatim (script-extracted with boundary guards and per-symbol assertions), nothing rewritten. Moved: the versioned-RPC plumbing (callVersionedRetrievalRpc, legacy owner/public merge, RPC error + variant-fanout telemetry), the lexical chunk / document-lookup / table-fact / embedding-field / index-unit candidate searchers, the chunk hydrators for memory cards and signal matches, and the memory-boost merge step. rag.ts (5,779 -> ~4,790 lines) imports the surfaces it orchestrates and re-exports callVersionedRetrievalRpc, loadChunksForMemoryCards, and loadChunksForSignalMatches so existing test imports are untouched. The extracted module depends only on non-rag lib modules - no import cycle. Verified: typecheck + lint clean, eval:rag:offline green (279 contract tests), verify:cheap green (2,301 tests), verify:pr-local green (build + client-bundle scan). The extracted module is byte-identical whether generated from the pre- or post-#626 base (the hardening PR touched no moved range). Co-Authored-By: Claude Fable 5 --- src/lib/rag-candidate-sources.ts | 1057 ++++++++++++++++++++++++++++++ src/lib/rag.ts | 1052 +---------------------------- 2 files changed, 1076 insertions(+), 1033 deletions(-) create mode 100644 src/lib/rag-candidate-sources.ts diff --git a/src/lib/rag-candidate-sources.ts b/src/lib/rag-candidate-sources.ts new file mode 100644 index 000000000..b62be11a5 --- /dev/null +++ b/src/lib/rag-candidate-sources.ts @@ -0,0 +1,1057 @@ +/** + * rag-candidate-sources - candidate retrieval surfaces for the RAG search + * waterfall, extracted verbatim from rag.ts (decomposition phase 1, + * 2026-07-14; zero behavior change). + * + * Contents: the versioned-RPC plumbing (callVersionedRetrievalRpc + legacy + * owner/public merge + RPC error/fan-out telemetry), the lexical chunk / + * document-lookup / table-fact / embedding-field / index-unit candidate + * searchers, the chunk hydrators for memory cards and signal matches, and + * the memory-boost merge step. rag.ts re-exports the symbols its tests and + * callers import from it. + */ +import { createAdminClient } from "@/lib/supabase/admin"; +import { + PUBLIC_OWNER_FILTER_SENTINEL, + retrievalAccessScopeForArgs, + retrievalAccessScopeKey, + retrievalRpcScopeArgs, + type RetrievalAccessScope, +} from "@/lib/owner-scope"; +import { + analyzeClinicalQuery, + buildClinicalTextSearchQuery, + normalizedClinicalSearchTokens, +} from "@/lib/clinical-search"; +import { applyMemoryCardBoosts, fetchMemoryCardsForQuery } from "@/lib/deep-memory"; +import { env } from "@/lib/env"; +import { logger } from "@/lib/logger"; +import { + firstVariantPoolIsStrong, + maxTextRpcQueryVariants, + relaxVariantToOrQuery, + shouldRelaxWeakTextMatches, +} from "@/lib/rag-retrieval-variants"; +import type { SearchTelemetry } from "@/lib/rag-contracts"; +import { committedIndexGeneration } from "@/lib/reindex-pipeline"; +import { isMissingRetrievalRpcError } from "@/lib/retrieval-rpc-rollout"; +import { normalizeSourceMetadata } from "@/lib/source-metadata"; +import { isReviewedTablePromotable } from "@/lib/table-review"; +import type { DocumentIndexUnitMatch, DocumentMemoryCard, SearchResult } from "@/lib/types"; + +// P0.1: a hybrid RPC returning an error (vs zero rows) means the whole layer silently degraded. +// Previously every call site did `if (error || !data?.length) return []` and dropped the error on +// the floor, which is how the live schema drift (42702) went unnoticed. Log it structurally and, +// where telemetry is in scope, record the failing RPC + code so it shows up in rag_retrieval_logs. +export type SupabaseRpcError = { message?: string; code?: string; details?: string; hint?: string } | null; + +function legacyRankFields(versionedName: string) { + if (versionedName === "match_document_chunks_v2") return ["similarity"]; + if (versionedName === "match_documents_for_query_v2") return ["text_rank"]; + if (versionedName.includes("_text_v2")) return ["hybrid_score", "text_rank"]; + return ["hybrid_score", "rrf_score", "similarity", "text_rank"]; +} + +function mergeLegacyAccessRows(ownerRows: T[], publicRows: T[], matchCount: unknown, rankFields: string[]): T[] { + const byKey = new Map(); + const rowRecord = (row: T) => (row && typeof row === "object" ? (row as Record) : {}); + const rowKey = (row: T) => String(rowRecord(row).id ?? rowRecord(row).document_id ?? ""); + const compare = (left: T, right: T) => { + const leftRecord = rowRecord(left); + const rightRecord = rowRecord(right); + for (const field of rankFields) { + const leftScore = typeof leftRecord[field] === "number" ? leftRecord[field] : Number.NEGATIVE_INFINITY; + const rightScore = typeof rightRecord[field] === "number" ? rightRecord[field] : Number.NEGATIVE_INFINITY; + if (leftScore !== rightScore) return rightScore - leftScore; + } + return rowKey(left).localeCompare(rowKey(right)); + }; + for (const row of [...ownerRows, ...publicRows]) { + const key = rowKey(row) || `unkeyed:${byKey.size}`; + const current = byKey.get(key); + if (!current || compare(row, current) < 0) byKey.set(key, row); + } + const merged = [...byKey.values()].sort(compare); + const limit = typeof matchCount === "number" ? Math.max(1, Math.min(matchCount, 100)) : merged.length; + return merged.slice(0, limit); +} + +export async function callVersionedRetrievalRpc( + supabase: ReturnType, + versionedName: string, + legacyName: string, + args: Record, +): Promise<{ data: T | null; error: SupabaseRpcError }> { + const client = supabase as unknown as { + rpc: (name: string, rpcArgs: Record) => Promise<{ data: T | null; error: SupabaseRpcError }>; + }; + const versioned = await client.rpc(versionedName, args); + if (versioned && !isMissingRetrievalRpcError(versioned.error)) return versioned; + const legacyArgs = { ...args }; + delete legacyArgs.include_public; + const ownerResult = await client.rpc(legacyName, legacyArgs); + const ownerFilter = String(args.owner_filter ?? ""); + if ( + ownerResult.error || + args.include_public !== true || + !ownerFilter || + ownerFilter === PUBLIC_OWNER_FILTER_SENTINEL + ) { + return ownerResult; + } + const publicResult = await client.rpc(legacyName, { + ...legacyArgs, + owner_filter: PUBLIC_OWNER_FILTER_SENTINEL, + }); + if (publicResult.error) return publicResult; + return { + data: mergeLegacyAccessRows( + ownerResult.data ?? [], + publicResult.data ?? [], + args.match_count, + legacyRankFields(versionedName), + ) as T, + error: null, + }; +} + +/** Record hybrid rpc error. */ +export function recordHybridRpcError(telemetry: SearchTelemetry | undefined, rpc: string, error: SupabaseRpcError) { + if (!error) return; + const code = error.code ?? "unknown"; + logger.error("hybrid_rpc_failed", { rpc, code, message: error.message, hint: error.hint }); + if (telemetry) { + telemetry.hybrid_rpc_errors = { + ...(telemetry.hybrid_rpc_errors ?? {}), + [rpc]: code, + }; + } +} + +/** Record how many variant RPCs a lexical surface actually issued (PT-02 early-exit). */ +function recordTextVariantFanout( + telemetry: SearchTelemetry | undefined, + rpc: string, + calls: number, + earlyExit: boolean, +) { + if (!telemetry) return; + telemetry.text_variant_rpc_calls = { + ...(telemetry.text_variant_rpc_calls ?? {}), + [rpc]: calls, + }; + if (earlyExit) telemetry.text_variant_early_exit = true; +} + +/** Merge search results. */ +export function mergeSearchResults(primary: SearchResult[], secondary: SearchResult[]) { + const merged = new Map(); + + for (const result of [...primary, ...secondary]) { + const existing = merged.get(result.id); + if (!existing) { + merged.set(result.id, result); + continue; + } + + const existingScore = existing.hybrid_score ?? existing.similarity; + const resultScore = result.hybrid_score ?? result.similarity; + merged.set(result.id, resultScore > existingScore ? result : existing); + } + + return Array.from(merged.values()); +} + +/** + * Retrieves lexical document chunk candidates for the supplied query variants. + * + * @param queryVariants - Query variants used for strict matching and recall-oriented fallback searches. + * @param matchCount - Maximum number of candidates requested for the primary query. + * @param telemetry - Optional retrieval telemetry updated with RPC failures and text-relaxation usage. + * @returns Matching document chunks, using corrected or relaxed queries when strict matching produces no candidates. + */ +export async function searchTextChunkCandidates(args: { + supabase: ReturnType; + queryVariants: string[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + allowGlobalSearch?: boolean; + matchCount: number; + telemetry?: SearchTelemetry; +}) { + const runChunkText = async (queryText: string, matchCount: number) => { + const accessScope = retrievalAccessScopeForArgs(args); + const { data, error } = await callVersionedRetrievalRpc( + args.supabase, + "match_document_chunks_text_v2", + "match_document_chunks_text", + { + query_text: queryText, + match_count: matchCount, + document_filters: args.documentIds ?? undefined, + ...retrievalRpcScopeArgs(accessScope), + }, + ); + // Report the error before returning empty so a schema drift on this + // most-terminal lexical layer surfaces in hybrid_rpc_errors telemetry + // instead of silently degrading to zero candidates. Return value unchanged. + if (error) recordHybridRpcError(args.telemetry, "match_document_chunks_text", error); + return error || !data?.length ? ([] as SearchResult[]) : (data as SearchResult[]); + }; + + const variants = args.queryVariants.slice(0, maxTextRpcQueryVariants); + const [primaryVariant, ...siblingVariants] = variants; + const primaryResults = primaryVariant === undefined ? [] : await runChunkText(primaryVariant, args.matchCount); + // PT-02: skip the near-duplicate sibling variants when the primary pool is + // already deep and anchored by a precise hit; weak/empty pools keep the full + // fan-out (one extra sequential hop) so recall is unchanged where it matters. + const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryResults, args.matchCount); + const resultSets = skipSiblings + ? [primaryResults] + : [ + primaryResults, + ...(await Promise.all(siblingVariants.map((variant) => runChunkText(variant, Math.min(args.matchCount, 32))))), + ]; + recordTextVariantFanout(args.telemetry, "match_document_chunks_text", resultSets.length, skipSiblings); + const merged = resultSets.reduce( + (accumulated, resultSet) => mergeSearchResults(resultSet, accumulated), + [] as SearchResult[], + ); + if (args.telemetry) args.telemetry.text_or_relaxation_used = "none"; + if (merged.length > 0) { + // P8b extension: strict-AND matched, but weakly. Append OR-relaxed recall behind the + // strict matches so a buried chunk can enter the candidate pool without displacing any + // precise match (mergeSearchResults keeps primary/strict precedence). Trigram correction + // is not run here — it exists to rescue *empty* strict retrieval (RC6), and correcting a + // query that already matched would second-guess a working query. + if (env.RAG_TEXT_WEAK_OR_RELAXATION && shouldRelaxWeakTextMatches(merged)) { + const weakRelaxed = relaxVariantToOrQuery(variants[0] ?? ""); + if (weakRelaxed) { + const orResults = await runChunkText(weakRelaxed, Math.min(args.matchCount, 24)); + if (orResults.length > 0) { + if (args.telemetry) args.telemetry.text_or_relaxation_used = "weak_augment"; + return mergeSearchResults(merged, orResults); + } + } + } + return merged; + } + + // Strict AND variants matched nothing. Two fallbacks, in order: + // (item 10, RC6) a typo the hard-coded map misses can block an otherwise-precise query — so first + // trigram-correct against the known clinical-term vocabulary and retry the corrected query + // STRICTLY. This must run before OR-relaxation: otherwise a query like "clozapin monitoring" + // would OR-match generic "monitoring" docs and never surface the intended clozapine result. + // (8b) then OR-relax the best query we have to recover recall for long multi-term queries. + // Both are reached only when every prior attempt was empty, so neither overrides a precise match. + const primary = variants[0] ?? ""; + let effectivePrimary = primary; + if (primary) { + const { data: corrected } = await args.supabase.rpc("correct_clinical_query_terms", { + input_query: primary, + min_sim: 0.45, + }); + if (typeof corrected === "string" && corrected && corrected !== primary) { + const correctedResults = await runChunkText(corrected, args.matchCount); + if (correctedResults.length > 0) return correctedResults; + effectivePrimary = corrected; + } + } + + const relaxed = relaxVariantToOrQuery(effectivePrimary); + if (relaxed) { + const relaxedResults = await runChunkText(relaxed, args.matchCount); + if (relaxedResults.length > 0) { + if (args.telemetry) args.telemetry.text_or_relaxation_used = "empty_fallback"; + return relaxedResults; + } + } + return merged; +} + +type DocumentLookupRow = { + id: string; + owner_id?: string | null; + title: string; + file_name: string; + status?: string; + page_count?: number; + chunk_count?: number; + image_count?: number; + metadata?: unknown; + text_rank?: number; + match_reason?: string; +}; + +type DocumentLookupChunkRow = { + id: string; + document_id: string; + page_number: number | null; + chunk_index: number; + section_heading: string | null; + section_path?: string[] | null; + heading_level?: number | null; + parent_heading?: string | null; + anchor_id?: string | null; + content: string; + retrieval_synopsis?: string | null; + image_ids: string[] | null; + text_rank?: number | null; +}; + +export type ChunkSignalMatch = { + chunkId: string; + similarity: number; + textRank: number; + hybridScore: number; + reason: string; + fieldType?: string | null; + tableFacts?: Array<{ + id: string; + document_id: string; + source_chunk_id: string | null; + source_image_id: string | null; + page_number: number | null; + table_title: string | null; + row_label: string | null; + clinical_parameter: string | null; + threshold_value: string | null; + action: string | null; + text_rank?: number | null; + match_reason?: string | null; + }>; + indexUnit?: DocumentIndexUnitMatch | null; +}; + +type IndexUnitRpcRow = DocumentIndexUnitMatch & { + document_id: string; + source_chunk_id: string | null; + similarity?: number | null; + text_rank?: number | null; + hybrid_score?: number | null; +}; + +type TableFactRpcRow = { + id: string; + document_id: string; + source_chunk_id: string | null; + source_image_id: string | null; + page_number: number | null; + table_title: string | null; + row_label: string | null; + clinical_parameter: string | null; + threshold_value: string | null; + action: string | null; + text_rank?: number | null; + match_reason?: string | null; + metadata?: Record | null; +}; + +/** Document lookup chunk terms. */ +function documentLookupChunkTerms(query: string) { + const shortClinicalTerms = new Set(["ed", "im", "po", "pt"]); + return normalizedClinicalSearchTokens(query) + .filter((term) => term.length >= 3 || shortClinicalTerms.has(term)) + .slice(0, 8); +} + +/** Document lookup chunk score. */ +function documentLookupChunkScore(chunk: DocumentLookupChunkRow, terms: string[]) { + if (terms.length === 0) return 0; + const heading = chunk.section_heading?.toLowerCase() ?? ""; + const content = `${chunk.retrieval_synopsis ?? ""} ${chunk.content}`.toLowerCase(); + const matched = terms.filter((term) => heading.includes(term) || content.includes(term)); + const coverage = matched.length / terms.length; + const headingHits = matched.filter((term) => heading.includes(term)).length; + return coverage + headingHits * 0.08 + Math.max(0, 0.08 - chunk.chunk_index * 0.0005); +} + +/** Fetch best document lookup chunks. */ +async function fetchBestDocumentLookupChunks(args: { + supabase: ReturnType; + documentIds: string[]; + query: string; + limit: number; + ownerId?: string; + accessScope?: RetrievalAccessScope; + allowGlobalSearch?: boolean; +}) { + const terms = documentLookupChunkTerms(args.query); + const { data: rpcChunks, error: rpcError } = await callVersionedRetrievalRpc( + args.supabase, + "match_document_lookup_chunks_text_v2", + "match_document_lookup_chunks_text", + { + query_text: args.query, + document_filters: args.documentIds ?? undefined, + match_count: Math.max(args.limit * 3, 24), + ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), + }, + ); + if (!rpcError && rpcChunks?.length) { + const ranked = (rpcChunks as DocumentLookupChunkRow[]) + .map((chunk) => ({ + chunk, + score: Math.max(Number(chunk.text_rank ?? 0), documentLookupChunkScore(chunk, terms)), + })) + .filter((item) => item.score > 0) + .sort((a, b) => b.score - a.score || a.chunk.chunk_index - b.chunk.chunk_index) + .map((item) => item.chunk); + if (ranked.length) return { chunks: ranked.slice(0, args.limit), terms }; + } + + const safeFilters = terms + .map((term) => term.replace(/[%_,]/g, " ").trim()) + .filter(Boolean) + .flatMap((term) => [ + `content.ilike.%${term}%`, + `retrieval_synopsis.ilike.%${term}%`, + `section_heading.ilike.%${term}%`, + ]) + .join(","); + const baseQuery = args.supabase + .from("document_chunks") + .select( + "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids", + ) + .in("document_id", args.documentIds) + .limit(Math.max(args.limit * 4, 24)); + + const { data: matchedChunks, error: matchedError } = safeFilters + ? await baseQuery.or(safeFilters) + : await baseQuery.order("chunk_index", { ascending: true }); + + if (!matchedError && matchedChunks?.length) { + const ranked = (matchedChunks as DocumentLookupChunkRow[]) + .map((chunk) => ({ chunk, score: documentLookupChunkScore(chunk, terms) })) + .filter((item) => item.score > 0) + .sort((a, b) => b.score - a.score || a.chunk.chunk_index - b.chunk.chunk_index) + .map((item) => item.chunk); + if (ranked.length) return { chunks: ranked.slice(0, args.limit), terms }; + } + + const fallbackQuery = args.supabase + .from("document_chunks") + .select( + "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids", + ) + .in("document_id", args.documentIds) + .order("chunk_index", { ascending: true }) + .limit(args.limit); + const { data: fallbackChunks, error: fallbackError } = await fallbackQuery; + if (fallbackError || !fallbackChunks?.length) return { chunks: [] as DocumentLookupChunkRow[], terms }; + return { chunks: fallbackChunks as DocumentLookupChunkRow[], terms }; +} + +/** Fetch document title alias rows. */ +async function fetchDocumentTitleAliasRows(args: { + supabase: ReturnType; + query: string; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; +}) { + const terms = analyzeClinicalQuery(args.query) + .documentTitleTerms.map((term) => term.replace(/[%_,]/g, " ").replace(/\s+/g, " ").trim()) + .filter((term) => term.length >= 4) + .slice(0, 8); + if (!terms.length) return [] as DocumentLookupRow[]; + + const filters = terms.flatMap((term) => [`title.ilike.%${term}%`, `file_name.ilike.%${term}%`]).join(","); + let query = args.supabase + .from("documents") + .select("id,owner_id,title,file_name,status,page_count,chunk_count,image_count,metadata"); + if (typeof (query as { or?: unknown }).or !== "function") return [] as DocumentLookupRow[]; + query = query.or(filters).eq("status", "indexed").limit(12); + const accessScope = retrievalAccessScopeForArgs(args); + if (accessScope.ownerId && accessScope.includePublic) { + query = query.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); + } else if (accessScope.ownerId) { + query = query.eq("owner_id", accessScope.ownerId); + } else { + query = query.is("owner_id", null); + } + if (args.documentIds?.length) query = query.in("id", args.documentIds); + + const { data, error } = await query; + if (error || !data?.length) return [] as DocumentLookupRow[]; + + return (data as DocumentLookupRow[]).map((document) => ({ + ...document, + text_rank: Math.max(Number(document.text_rank ?? 0), 0.34), + match_reason: document.match_reason ?? "title_alias", + })); +} + +/** + * Retrieves and ranks document chunks for document-focused queries. + * + * @param args - Search configuration, including the required owner scope and optional document restrictions. + * @returns Ranked document lookup results, limited to `matchCount`. + */ +export async function searchDocumentLookupFastPath(args: { + supabase: ReturnType; + query: string; + queryVariants?: string[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + matchCount: number; + telemetry?: SearchTelemetry; +}): Promise { + if (!args.ownerId) return [] as SearchResult[]; + const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( + 0, + maxTextRpcQueryVariants, + ); + const runDocumentLookup = async (variant: string, matchCount: number) => { + const { data, error } = await callVersionedRetrievalRpc( + args.supabase, + "match_documents_for_query_v2", + "match_documents_for_query", + { + query_text: variant, + match_count: matchCount, + ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), + }, + ); + if (error) recordHybridRpcError(args.telemetry, "match_documents_for_query", error); + if (error || !data?.length) return [] as DocumentLookupRow[]; + return data as DocumentLookupRow[]; + }; + const [primaryVariant, ...siblingVariants] = variants; + const primaryDocuments = primaryVariant === undefined ? [] : await runDocumentLookup(primaryVariant, 12); + const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryDocuments, 12); + const documentSets = skipSiblings + ? [primaryDocuments] + : [primaryDocuments, ...(await Promise.all(siblingVariants.map((variant) => runDocumentLookup(variant, 8))))]; + recordTextVariantFanout(args.telemetry, "match_documents_for_query", documentSets.length, skipSiblings); + const titleAliasDocuments = await fetchDocumentTitleAliasRows({ + supabase: args.supabase, + query: args.query, + ownerId: args.ownerId, + accessScope: args.accessScope, + documentIds: args.documentIds, + }); + const documentsById = new Map(); + for (const document of [...titleAliasDocuments, ...documentSets.flat()]) { + const existing = documentsById.get(document.id); + if (!existing || Number(document.text_rank ?? 0) > Number(existing.text_rank ?? 0)) { + documentsById.set(document.id, document); + } + } + const documents = Array.from(documentsById.values()); + if (!documents.length) return []; + + const allowedDocuments = args.documentIds?.length ? new Set(args.documentIds) : null; + const rankedDocuments = (documents as DocumentLookupRow[]) + .filter((document) => !allowedDocuments || allowedDocuments.has(document.id)) + .map((document) => ({ + document, + score: Math.min(0.34, Number(document.text_rank ?? 0)), + })) + .filter((item) => item.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, 5); + + if (rankedDocuments.length === 0) return []; + + const documentById = new Map(rankedDocuments.map((item) => [item.document.id, item.document])); + const scoreByDocument = new Map(rankedDocuments.map((item) => [item.document.id, item.score])); + const { chunks, terms } = await fetchBestDocumentLookupChunks({ + supabase: args.supabase, + documentIds: rankedDocuments.map((item) => item.document.id), + query: args.query, + limit: Math.max(args.matchCount, rankedDocuments.length * 4), + ownerId: args.ownerId, + accessScope: args.accessScope, + }); + + if (!chunks.length) return []; + + const results: SearchResult[] = []; + for (const chunk of chunks) { + const document = documentById.get(chunk.document_id); + if (!document) continue; + const documentScore = scoreByDocument.get(chunk.document_id) ?? 0; + const chunkScore = documentLookupChunkScore(chunk, terms); + // Not a cosine: fabricated from title/label match strength (RC9 — tagged below). + const similarity = Math.min(0.92, 0.58 + documentScore + Math.min(0.12, chunkScore * 0.08)); + results.push({ + similarity_origin: "synthetic_text", + id: chunk.id, + document_id: chunk.document_id, + title: document.title, + file_name: document.file_name, + page_number: chunk.page_number, + chunk_index: chunk.chunk_index, + section_heading: chunk.section_heading, + section_path: chunk.section_path ?? [], + heading_level: chunk.heading_level ?? null, + parent_heading: chunk.parent_heading ?? null, + anchor_id: chunk.anchor_id ?? null, + content: chunk.content, + retrieval_synopsis: chunk.retrieval_synopsis ?? null, + image_ids: chunk.image_ids ?? [], + source_metadata: normalizeSourceMetadata(document.metadata), + similarity, + text_rank: documentScore, + hybrid_score: Math.min(0.94, similarity + 0.02), + images: [], + }); + } + + return results + .sort( + (a, b) => (b.hybrid_score ?? b.similarity) - (a.hybrid_score ?? a.similarity) || a.chunk_index - b.chunk_index, + ) + .slice(0, args.matchCount); +} + +/** Memory card chunk score. */ +export function memoryCardChunkScore(card: DocumentMemoryCard) { + const hybridScore = Number(card.metadata?.memory_hybrid_score); + if (Number.isFinite(hybridScore) && hybridScore > 0) return Math.min(1, hybridScore); + return Math.min(1, card.confidence ?? 0.5); +} + +/** Load chunks for memory cards. */ +export async function loadChunksForMemoryCards( + supabase: ReturnType, + cards: DocumentMemoryCard[], + accessScope: RetrievalAccessScope, +) { + const documentIds = Array.from(new Set(cards.map((card) => card.document_id))).slice(0, 80); + if (documentIds.length === 0) return [] as SearchResult[]; + let documentQuery = supabase + .from("documents") + .select("id,title,file_name,metadata,owner_id,status") + .in("id", documentIds) + .eq("status", "indexed"); + if (accessScope.ownerId && accessScope.includePublic) { + documentQuery = documentQuery.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); + } else if (accessScope.ownerId) { + documentQuery = documentQuery.eq("owner_id", accessScope.ownerId); + } else { + documentQuery = documentQuery.is("owner_id", null); + } + const { data: documents, error: documentsError } = await documentQuery; + if (documentsError || !documents?.length) return [] as SearchResult[]; + + const documentById = new Map(documents.map((document) => [document.id, document])); + const allowedDocumentIds = new Set(documentById.keys()); + const chunkIds = Array.from( + new Set( + cards.filter((card) => allowedDocumentIds.has(card.document_id)).flatMap((card) => card.source_chunk_ids ?? []), + ), + ).slice(0, 80); + if (chunkIds.length === 0) return [] as SearchResult[]; + const { data: chunks, error: chunksError } = await supabase + .from("document_chunks") + .select( + "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids,index_generation_id", + ) + .in("id", chunkIds) + .in("document_id", [...allowedDocumentIds]) + .limit(chunkIds.length); + if (chunksError || !chunks?.length) return [] as SearchResult[]; + const bestCardByChunk = new Map(); + for (const card of cards) { + for (const chunkId of card.source_chunk_ids ?? []) { + const existing = bestCardByChunk.get(chunkId); + if (!existing || memoryCardChunkScore(card) > memoryCardChunkScore(existing)) bestCardByChunk.set(chunkId, card); + } + } + + return chunks + .map((chunk) => { + const document = documentById.get(chunk.document_id); + if (!document) return null; + const committedGeneration = committedIndexGeneration(document.metadata); + if (chunk.index_generation_id && chunk.index_generation_id !== committedGeneration) return null; + const card = bestCardByChunk.get(chunk.id); + // Not a cosine: fabricated from memory-card confidence (RC9 — tagged below). + const similarity = Math.min(0.92, 0.58 + (card?.confidence ?? 0.5) * 0.28); + return { + similarity_origin: "synthetic_text" as const, + id: chunk.id, + document_id: chunk.document_id, + title: document.title, + file_name: document.file_name, + page_number: chunk.page_number, + chunk_index: chunk.chunk_index, + section_heading: chunk.section_heading, + section_path: chunk.section_path ?? [], + heading_level: chunk.heading_level ?? null, + parent_heading: chunk.parent_heading ?? null, + anchor_id: chunk.anchor_id ?? null, + content: chunk.content, + retrieval_synopsis: chunk.retrieval_synopsis ?? null, + image_ids: chunk.image_ids ?? [], + source_metadata: normalizeSourceMetadata(document.metadata), + similarity, + text_rank: card?.confidence ?? 0, + hybrid_score: Math.min(0.96, similarity + 0.03), + images: [], + } satisfies SearchResult; + }) + .filter(Boolean) as SearchResult[]; +} + +/** Load chunks for signal matches. */ +export async function loadChunksForSignalMatches(args: { + supabase: ReturnType; + matches: ChunkSignalMatch[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; +}) { + const bestMatchByChunk = new Map(); + for (const match of args.matches) { + const existing = bestMatchByChunk.get(match.chunkId); + if (!existing || match.hybridScore > existing.hybridScore) bestMatchByChunk.set(match.chunkId, match); + } + const chunkIds = Array.from(bestMatchByChunk.keys()).slice(0, 80); + if (chunkIds.length === 0) return [] as SearchResult[]; + + const accessScope = retrievalAccessScopeForArgs(args); + const { data: chunkScopes, error: chunkScopesError } = await args.supabase + .from("document_chunks") + .select("id,document_id") + .in("id", chunkIds) + .limit(chunkIds.length); + if (chunkScopesError || !chunkScopes?.length) return [] as SearchResult[]; + + const documentIds = Array.from(new Set(chunkScopes.map((chunk) => chunk.document_id))); + let documentQuery = args.supabase + .from("documents") + .select("id,title,file_name,metadata,owner_id,status") + .in("id", documentIds) + .eq("status", "indexed"); + if (accessScope.ownerId && accessScope.includePublic) { + documentQuery = documentQuery.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); + } else if (accessScope.ownerId) { + documentQuery = documentQuery.eq("owner_id", accessScope.ownerId); + } else { + documentQuery = documentQuery.is("owner_id", null); + } + const { data: documents, error: documentsError } = await documentQuery; + if (documentsError || !documents?.length) return [] as SearchResult[]; + const documentById = new Map(documents.map((document) => [document.id, document])); + const allowedDocumentIds = new Set(documentById.keys()); + const allowedChunkIds = chunkScopes + .filter((chunk) => allowedDocumentIds.has(chunk.document_id)) + .map((chunk) => chunk.id); + if (allowedChunkIds.length === 0) return [] as SearchResult[]; + const { data: chunks, error: chunksError } = await args.supabase + .from("document_chunks") + .select( + "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids,index_generation_id", + ) + .in("id", allowedChunkIds) + .in("document_id", [...allowedDocumentIds]) + .limit(allowedChunkIds.length); + if (chunksError || !chunks?.length) return [] as SearchResult[]; + + return chunks + .map((chunk) => { + const document = documentById.get(chunk.document_id); + const match = bestMatchByChunk.get(chunk.id); + if (!document || !match) return null; + const committedGeneration = committedIndexGeneration(document.metadata); + if (chunk.index_generation_id && chunk.index_generation_id !== committedGeneration) return null; + return { + id: chunk.id, + document_id: chunk.document_id, + title: document.title, + file_name: document.file_name, + page_number: chunk.page_number, + chunk_index: chunk.chunk_index, + section_heading: chunk.section_heading, + section_path: chunk.section_path ?? [], + heading_level: chunk.heading_level ?? null, + parent_heading: chunk.parent_heading ?? null, + anchor_id: chunk.anchor_id ?? null, + content: chunk.content, + retrieval_synopsis: chunk.retrieval_synopsis ?? null, + image_ids: chunk.image_ids ?? [], + source_metadata: normalizeSourceMetadata(document.metadata), + // ChunkSignalMatch similarities are fabricated from table-fact text rank (RC9). + similarity_origin: "synthetic_text" as const, + similarity: match.similarity, + text_rank: match.textRank, + hybrid_score: match.hybridScore, + table_facts: match.tableFacts?.filter((fact) => + isReviewedTablePromotable((fact as { metadata?: unknown }).metadata), + ), + index_unit: match.indexUnit ?? null, + match_explanation: { + sectionHit: match.reason === "section_context" || match.indexUnit?.unit_type === "section_summary", + tableHit: match.reason.startsWith("table") || match.indexUnit?.unit_type === "table_fact", + indexUnitType: match.indexUnit?.unit_type ?? null, + vectorSimilarity: match.similarity, + textRank: match.textRank, + fieldType: match.fieldType ?? null, + freshness: normalizeSourceMetadata(document.metadata).document_status, + extractionQuality: normalizeSourceMetadata(document.metadata).extraction_quality, + reasons: [match.reason, match.indexUnit?.unit_type ? `unit:${match.indexUnit.unit_type}` : ""].filter( + Boolean, + ), + }, + images: [], + } satisfies SearchResult; + }) + .filter(Boolean) as SearchResult[]; +} + +/** + * Retrieves document chunks containing table facts relevant to a query. + * + * @param args - Retrieval options, including query variants, scope filters, result limit, and optional telemetry. + * @returns Search results containing the highest-ranked table facts grouped by source chunk. + */ +export async function searchTableFactCandidates(args: { + supabase: ReturnType; + query: string; + queryVariants?: string[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + allowGlobalSearch?: boolean; + matchCount: number; + telemetry?: SearchTelemetry; +}) { + const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( + 0, + maxTextRpcQueryVariants, + ); + const runTableFacts = async (variant: string, matchCount: number) => { + const { data, error } = await callVersionedRetrievalRpc( + args.supabase, + "match_document_table_facts_text_v2", + "match_document_table_facts_text", + { + query_text: variant, + match_count: matchCount, + document_filters: args.documentIds ?? undefined, + ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), + }, + ); + if (error) recordHybridRpcError(args.telemetry, "match_document_table_facts_text", error); + if (error || !data?.length) return [] as TableFactRpcRow[]; + return data as TableFactRpcRow[]; + }; + const [primaryVariant, ...siblingVariants] = variants; + const primaryFacts = primaryVariant === undefined ? [] : await runTableFacts(primaryVariant, args.matchCount); + const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryFacts, args.matchCount); + const factSets = skipSiblings + ? [primaryFacts] + : [ + primaryFacts, + ...(await Promise.all(siblingVariants.map((variant) => runTableFacts(variant, Math.min(args.matchCount, 24))))), + ]; + recordTextVariantFanout(args.telemetry, "match_document_table_facts_text", factSets.length, skipSiblings); + const data = factSets.flat(); + if (!data.length) return [] as SearchResult[]; + + const grouped = new Map(); + for (const fact of data) { + if (!fact.source_chunk_id) continue; + const textRank = Number(fact.text_rank ?? 0); + const current = grouped.get(fact.source_chunk_id); + const tableFact = { ...fact, text_rank: textRank, match_reason: fact.match_reason ?? "table_row" }; + const next: ChunkSignalMatch = { + chunkId: fact.source_chunk_id, + similarity: Math.min(0.94, 0.62 + Math.min(textRank, 1) * 0.3), + textRank, + hybridScore: Math.min(0.97, 0.66 + Math.min(textRank, 1) * 0.3), + reason: fact.match_reason ?? "table_row", + tableFacts: [...(current?.tableFacts ?? []), tableFact].slice(0, 5), + }; + if (!current || next.hybridScore > current.hybridScore) grouped.set(fact.source_chunk_id, next); + } + + return loadChunksForSignalMatches({ + supabase: args.supabase, + matches: Array.from(grouped.values()), + ownerId: args.ownerId, + accessScope: args.accessScope, + }); +} + +/** Search embedding field candidates. */ +export async function searchEmbeddingFieldCandidates(args: { + supabase: ReturnType; + query: string; + queryEmbedding: number[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + allowGlobalSearch?: boolean; + matchCount: number; + telemetry?: SearchTelemetry; +}) { + const { data, error } = await callVersionedRetrievalRpc( + args.supabase, + "match_document_embedding_fields_hybrid_v2", + "match_document_embedding_fields_hybrid", + { + query_embedding: args.queryEmbedding as unknown as string, + query_text: buildClinicalTextSearchQuery(args.query), + match_count: args.matchCount, + min_similarity: 0.12, + document_filters: args.documentIds ?? undefined, + ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), + }, + ); + if (error) recordHybridRpcError(args.telemetry, "match_document_embedding_fields_hybrid", error); + if (error || !data?.length) return [] as SearchResult[]; + const matches = ( + data as Array<{ + source_chunk_id: string | null; + field_type: string | null; + similarity?: number | null; + text_rank?: number | null; + hybrid_score?: number | null; + }> + ) + .filter( + ( + row, + ): row is { + source_chunk_id: string; + field_type: string | null; + similarity?: number | null; + text_rank?: number | null; + hybrid_score?: number | null; + } => Boolean(row.source_chunk_id), + ) + .map((row) => ({ + chunkId: row.source_chunk_id, + similarity: Number(row.similarity ?? 0), + textRank: Number(row.text_rank ?? 0), + hybridScore: Number(row.hybrid_score ?? row.similarity ?? 0), + reason: "section_context", + fieldType: row.field_type, + })); + return loadChunksForSignalMatches({ + supabase: args.supabase, + matches, + ownerId: args.ownerId, + accessScope: args.accessScope, + }); +} + +/** Search index unit candidates. */ +export async function searchIndexUnitCandidates(args: { + supabase: ReturnType; + query: string; + queryEmbedding: number[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + allowGlobalSearch?: boolean; + matchCount: number; + telemetry?: SearchTelemetry; +}) { + const { data, error } = await callVersionedRetrievalRpc( + args.supabase, + "match_document_index_units_hybrid_v2", + "match_document_index_units_hybrid", + { + query_embedding: args.queryEmbedding as unknown as string, + query_text: buildClinicalTextSearchQuery(args.query), + match_count: args.matchCount, + min_similarity: 0.1, + document_filters: args.documentIds ?? undefined, + ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), + }, + ); + if (error) recordHybridRpcError(args.telemetry, "match_document_index_units_hybrid", error); + if (error || !data?.length) return [] as SearchResult[]; + const matches = (data as IndexUnitRpcRow[]) + .filter((row): row is IndexUnitRpcRow & { source_chunk_id: string } => Boolean(row.source_chunk_id)) + .map((row) => ({ + chunkId: row.source_chunk_id, + similarity: Number(row.similarity ?? 0), + textRank: Number(row.text_rank ?? 0), + hybridScore: Number(row.hybrid_score ?? row.similarity ?? 0), + reason: `index_unit:${row.unit_type}`, + fieldType: row.unit_type, + indexUnit: { + id: row.id, + document_id: row.document_id, + unit_type: row.unit_type, + title: row.title, + content: row.content, + source_chunk_id: row.source_chunk_id, + source_image_id: row.source_image_id, + page_start: row.page_start, + page_end: row.page_end, + heading_path: row.heading_path ?? [], + normalized_terms: row.normalized_terms ?? [], + source_span: row.source_span ?? null, + quality_score: row.quality_score, + extraction_mode: row.extraction_mode, + similarity: row.similarity, + text_rank: row.text_rank, + hybrid_score: row.hybrid_score, + metadata: row.metadata ?? null, + }, + })); + return loadChunksForSignalMatches({ + supabase: args.supabase, + matches, + ownerId: args.ownerId, + accessScope: args.accessScope, + }); +} + +export type MemoryCardCache = Map>; + +/** With memory boosted candidates. */ +export async function withMemoryBoostedCandidates(args: { + supabase: ReturnType; + query: string; + candidates: SearchResult[]; + queryEmbedding?: number[]; + ownerId?: string; + accessScope?: RetrievalAccessScope; + documentIds?: string[]; + matchCount: number; + cardCache?: MemoryCardCache; +}) { + // A3: the memory-card fetch is invoked at several waterfall stages. Memoize per request, + // scoped by owner/document filters because fetchMemoryCardsForQuery applies those filters. + const effectiveMatchCount = Math.max(args.matchCount, 48); + const documentScope = args.documentIds?.length ? [...args.documentIds].sort().join(",") : "all-documents"; + const cacheKey = [ + retrievalAccessScopeKey(retrievalAccessScopeForArgs(args)), + documentScope, + args.query, + args.queryEmbedding?.length ? "vec" : "text", + effectiveMatchCount, + ].join("\0"); + let cardsPromise = args.cardCache?.get(cacheKey); + if (!cardsPromise) { + cardsPromise = fetchMemoryCardsForQuery({ + supabase: args.supabase, + query: args.query, + queryEmbedding: args.queryEmbedding, + ownerId: args.ownerId, + accessScope: args.accessScope, + documentIds: args.documentIds, + matchCount: effectiveMatchCount, + }); + args.cardCache?.set(cacheKey, cardsPromise); + } + const cards = await cardsPromise; + if (cards.length === 0) return { results: args.candidates, cards }; + + const memoryChunkResults = await loadChunksForMemoryCards(args.supabase, cards, retrievalAccessScopeForArgs(args)); + const merged = mergeSearchResults(memoryChunkResults, args.candidates); + return { + results: applyMemoryCardBoosts(args.query, merged, cards), + cards, + }; +} diff --git a/src/lib/rag.ts b/src/lib/rag.ts index d4d6a2592..7ebc5c23c 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1,11 +1,23 @@ import { createAdminClient } from "@/lib/supabase/admin"; +import { retrievalAccessScopeForArgs, retrievalRpcScopeArgs } from "@/lib/owner-scope"; import { - PUBLIC_OWNER_FILTER_SENTINEL, - retrievalAccessScopeForArgs, - retrievalAccessScopeKey, - retrievalRpcScopeArgs, - type RetrievalAccessScope, -} from "@/lib/owner-scope"; + callVersionedRetrievalRpc, + memoryCardChunkScore, + mergeSearchResults, + recordHybridRpcError, + searchDocumentLookupFastPath, + searchEmbeddingFieldCandidates, + searchIndexUnitCandidates, + searchTableFactCandidates, + searchTextChunkCandidates, + withMemoryBoostedCandidates, + type MemoryCardCache, +} from "@/lib/rag-candidate-sources"; +export { + callVersionedRetrievalRpc, + loadChunksForMemoryCards, + loadChunksForSignalMatches, +} from "@/lib/rag-candidate-sources"; import { classifyCorpusGrounding } from "@/lib/corpus-grounding"; import type { Database, Json } from "@/lib/supabase/database.types"; import { @@ -69,14 +81,10 @@ import { assertGlobalSearchAllowed, buildRetrievalQueryVariants, fetchEnabledRagAliases, - firstVariantPoolIsStrong, - maxTextRpcQueryVariants, normalizeRetrievalVariant, ownerScopeForDocumentFilteredRetrieval, - relaxVariantToOrQuery, selectRagAliasExpansions, shouldApplyUnsupportedSearchShortCircuit, - shouldRelaxWeakTextMatches, textCandidateBudgetForQueryClass, } from "@/lib/rag-retrieval-variants"; export { @@ -131,7 +139,6 @@ import { } from "@/lib/clinical-search"; import { env, requestedOpenAIAnswerModels } from "@/lib/env"; import { ragAnswerPromptVersion, ragQueryClassifierPromptVersion, ragSummaryPromptVersion } from "@/lib/rag-versioning"; -import { logger } from "@/lib/logger"; import { captureServerEvent } from "@/lib/observability/error-capture"; import { answerPrivacyMetadata, @@ -140,7 +147,6 @@ import { queryTextForStorage, } from "@/lib/query-privacy"; import { normalizeSourceMetadata } from "@/lib/source-metadata"; -import { isReviewedTablePromotable } from "@/lib/table-review"; import { isClinicalImageEvidence, normalizeImageBbox } from "@/lib/image-filtering"; import { SOURCE_BACKED_REVIEW_FALLBACK_REASON, @@ -151,7 +157,7 @@ import { } from "@/lib/rag-routing"; import { fetchRelatedDocumentMetadata, fetchRelatedDocuments } from "@/lib/document-enrichment"; import { boldHighYieldClinicalText, boldRagAnswerHighYieldText, rankAnswerEvidence } from "@/lib/answer-ranking"; -import { applyMemoryCardBoosts, fetchMemoryCardsForQuery, ragDeepMemoryVersion } from "@/lib/deep-memory"; +import { ragDeepMemoryVersion } from "@/lib/deep-memory"; import { buildAnswerScoreExplanations, buildIndexingQuality, @@ -202,7 +208,6 @@ import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { clinicalModePrompt, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline"; -import { isMissingRetrievalRpcError } from "@/lib/retrieval-rpc-rollout"; import { buildRetrievalIntent, selectRetrievalEvidence } from "@/lib/retrieval-selection"; import { rankingConfig } from "@/lib/ranking-config"; import { z } from "zod"; @@ -225,8 +230,6 @@ import type { ClinicalImageUseClass, Citation, ClinicalQueryAnalysis, - DocumentIndexUnitMatch, - DocumentMemoryCard, EvidenceRelevance, RelatedDocument, OpenAITokenUsage, @@ -513,110 +516,6 @@ function recordRetrievalLayer( } } -// P0.1: a hybrid RPC returning an error (vs zero rows) means the whole layer silently degraded. -// Previously every call site did `if (error || !data?.length) return []` and dropped the error on -// the floor, which is how the live schema drift (42702) went unnoticed. Log it structurally and, -// where telemetry is in scope, record the failing RPC + code so it shows up in rag_retrieval_logs. -type SupabaseRpcError = { message?: string; code?: string; details?: string; hint?: string } | null; - -function legacyRankFields(versionedName: string) { - if (versionedName === "match_document_chunks_v2") return ["similarity"]; - if (versionedName === "match_documents_for_query_v2") return ["text_rank"]; - if (versionedName.includes("_text_v2")) return ["hybrid_score", "text_rank"]; - return ["hybrid_score", "rrf_score", "similarity", "text_rank"]; -} - -function mergeLegacyAccessRows(ownerRows: T[], publicRows: T[], matchCount: unknown, rankFields: string[]): T[] { - const byKey = new Map(); - const rowRecord = (row: T) => (row && typeof row === "object" ? (row as Record) : {}); - const rowKey = (row: T) => String(rowRecord(row).id ?? rowRecord(row).document_id ?? ""); - const compare = (left: T, right: T) => { - const leftRecord = rowRecord(left); - const rightRecord = rowRecord(right); - for (const field of rankFields) { - const leftScore = typeof leftRecord[field] === "number" ? leftRecord[field] : Number.NEGATIVE_INFINITY; - const rightScore = typeof rightRecord[field] === "number" ? rightRecord[field] : Number.NEGATIVE_INFINITY; - if (leftScore !== rightScore) return rightScore - leftScore; - } - return rowKey(left).localeCompare(rowKey(right)); - }; - for (const row of [...ownerRows, ...publicRows]) { - const key = rowKey(row) || `unkeyed:${byKey.size}`; - const current = byKey.get(key); - if (!current || compare(row, current) < 0) byKey.set(key, row); - } - const merged = [...byKey.values()].sort(compare); - const limit = typeof matchCount === "number" ? Math.max(1, Math.min(matchCount, 100)) : merged.length; - return merged.slice(0, limit); -} - -export async function callVersionedRetrievalRpc( - supabase: ReturnType, - versionedName: string, - legacyName: string, - args: Record, -): Promise<{ data: T | null; error: SupabaseRpcError }> { - const client = supabase as unknown as { - rpc: (name: string, rpcArgs: Record) => Promise<{ data: T | null; error: SupabaseRpcError }>; - }; - const versioned = await client.rpc(versionedName, args); - if (versioned && !isMissingRetrievalRpcError(versioned.error)) return versioned; - const legacyArgs = { ...args }; - delete legacyArgs.include_public; - const ownerResult = await client.rpc(legacyName, legacyArgs); - const ownerFilter = String(args.owner_filter ?? ""); - if ( - ownerResult.error || - args.include_public !== true || - !ownerFilter || - ownerFilter === PUBLIC_OWNER_FILTER_SENTINEL - ) { - return ownerResult; - } - const publicResult = await client.rpc(legacyName, { - ...legacyArgs, - owner_filter: PUBLIC_OWNER_FILTER_SENTINEL, - }); - if (publicResult.error) return publicResult; - return { - data: mergeLegacyAccessRows( - ownerResult.data ?? [], - publicResult.data ?? [], - args.match_count, - legacyRankFields(versionedName), - ) as T, - error: null, - }; -} - -/** Record hybrid rpc error. */ -function recordHybridRpcError(telemetry: SearchTelemetry | undefined, rpc: string, error: SupabaseRpcError) { - if (!error) return; - const code = error.code ?? "unknown"; - logger.error("hybrid_rpc_failed", { rpc, code, message: error.message, hint: error.hint }); - if (telemetry) { - telemetry.hybrid_rpc_errors = { - ...(telemetry.hybrid_rpc_errors ?? {}), - [rpc]: code, - }; - } -} - -/** Record how many variant RPCs a lexical surface actually issued (PT-02 early-exit). */ -function recordTextVariantFanout( - telemetry: SearchTelemetry | undefined, - rpc: string, - calls: number, - earlyExit: boolean, -) { - if (!telemetry) return; - telemetry.text_variant_rpc_calls = { - ...(telemetry.text_variant_rpc_calls ?? {}), - [rpc]: calls, - }; - if (earlyExit) telemetry.text_variant_early_exit = true; -} - /** Record search score telemetry. */ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) { if (!results.length) { @@ -1558,472 +1457,6 @@ async function logRagQuery(row: RagQueryInsert) { void insertRagQuery(row).catch(() => undefined); } -/** Merge search results. */ -function mergeSearchResults(primary: SearchResult[], secondary: SearchResult[]) { - const merged = new Map(); - - for (const result of [...primary, ...secondary]) { - const existing = merged.get(result.id); - if (!existing) { - merged.set(result.id, result); - continue; - } - - const existingScore = existing.hybrid_score ?? existing.similarity; - const resultScore = result.hybrid_score ?? result.similarity; - merged.set(result.id, resultScore > existingScore ? result : existing); - } - - return Array.from(merged.values()); -} - -/** - * Retrieves lexical document chunk candidates for the supplied query variants. - * - * @param queryVariants - Query variants used for strict matching and recall-oriented fallback searches. - * @param matchCount - Maximum number of candidates requested for the primary query. - * @param telemetry - Optional retrieval telemetry updated with RPC failures and text-relaxation usage. - * @returns Matching document chunks, using corrected or relaxed queries when strict matching produces no candidates. - */ -async function searchTextChunkCandidates(args: { - supabase: ReturnType; - queryVariants: string[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - allowGlobalSearch?: boolean; - matchCount: number; - telemetry?: SearchTelemetry; -}) { - const runChunkText = async (queryText: string, matchCount: number) => { - const accessScope = retrievalAccessScopeForArgs(args); - const { data, error } = await callVersionedRetrievalRpc( - args.supabase, - "match_document_chunks_text_v2", - "match_document_chunks_text", - { - query_text: queryText, - match_count: matchCount, - document_filters: args.documentIds ?? undefined, - ...retrievalRpcScopeArgs(accessScope), - }, - ); - // Report the error before returning empty so a schema drift on this - // most-terminal lexical layer surfaces in hybrid_rpc_errors telemetry - // instead of silently degrading to zero candidates. Return value unchanged. - if (error) recordHybridRpcError(args.telemetry, "match_document_chunks_text", error); - return error || !data?.length ? ([] as SearchResult[]) : (data as SearchResult[]); - }; - - const variants = args.queryVariants.slice(0, maxTextRpcQueryVariants); - const [primaryVariant, ...siblingVariants] = variants; - const primaryResults = primaryVariant === undefined ? [] : await runChunkText(primaryVariant, args.matchCount); - // PT-02: skip the near-duplicate sibling variants when the primary pool is - // already deep and anchored by a precise hit; weak/empty pools keep the full - // fan-out (one extra sequential hop) so recall is unchanged where it matters. - const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryResults, args.matchCount); - const resultSets = skipSiblings - ? [primaryResults] - : [ - primaryResults, - ...(await Promise.all(siblingVariants.map((variant) => runChunkText(variant, Math.min(args.matchCount, 32))))), - ]; - recordTextVariantFanout(args.telemetry, "match_document_chunks_text", resultSets.length, skipSiblings); - const merged = resultSets.reduce( - (accumulated, resultSet) => mergeSearchResults(resultSet, accumulated), - [] as SearchResult[], - ); - if (args.telemetry) args.telemetry.text_or_relaxation_used = "none"; - if (merged.length > 0) { - // P8b extension: strict-AND matched, but weakly. Append OR-relaxed recall behind the - // strict matches so a buried chunk can enter the candidate pool without displacing any - // precise match (mergeSearchResults keeps primary/strict precedence). Trigram correction - // is not run here — it exists to rescue *empty* strict retrieval (RC6), and correcting a - // query that already matched would second-guess a working query. - if (env.RAG_TEXT_WEAK_OR_RELAXATION && shouldRelaxWeakTextMatches(merged)) { - const weakRelaxed = relaxVariantToOrQuery(variants[0] ?? ""); - if (weakRelaxed) { - const orResults = await runChunkText(weakRelaxed, Math.min(args.matchCount, 24)); - if (orResults.length > 0) { - if (args.telemetry) args.telemetry.text_or_relaxation_used = "weak_augment"; - return mergeSearchResults(merged, orResults); - } - } - } - return merged; - } - - // Strict AND variants matched nothing. Two fallbacks, in order: - // (item 10, RC6) a typo the hard-coded map misses can block an otherwise-precise query — so first - // trigram-correct against the known clinical-term vocabulary and retry the corrected query - // STRICTLY. This must run before OR-relaxation: otherwise a query like "clozapin monitoring" - // would OR-match generic "monitoring" docs and never surface the intended clozapine result. - // (8b) then OR-relax the best query we have to recover recall for long multi-term queries. - // Both are reached only when every prior attempt was empty, so neither overrides a precise match. - const primary = variants[0] ?? ""; - let effectivePrimary = primary; - if (primary) { - const { data: corrected } = await args.supabase.rpc("correct_clinical_query_terms", { - input_query: primary, - min_sim: 0.45, - }); - if (typeof corrected === "string" && corrected && corrected !== primary) { - const correctedResults = await runChunkText(corrected, args.matchCount); - if (correctedResults.length > 0) return correctedResults; - effectivePrimary = corrected; - } - } - - const relaxed = relaxVariantToOrQuery(effectivePrimary); - if (relaxed) { - const relaxedResults = await runChunkText(relaxed, args.matchCount); - if (relaxedResults.length > 0) { - if (args.telemetry) args.telemetry.text_or_relaxation_used = "empty_fallback"; - return relaxedResults; - } - } - return merged; -} - -type DocumentLookupRow = { - id: string; - owner_id?: string | null; - title: string; - file_name: string; - status?: string; - page_count?: number; - chunk_count?: number; - image_count?: number; - metadata?: unknown; - text_rank?: number; - match_reason?: string; -}; - -type DocumentLookupChunkRow = { - id: string; - document_id: string; - page_number: number | null; - chunk_index: number; - section_heading: string | null; - section_path?: string[] | null; - heading_level?: number | null; - parent_heading?: string | null; - anchor_id?: string | null; - content: string; - retrieval_synopsis?: string | null; - image_ids: string[] | null; - text_rank?: number | null; -}; - -type ChunkSignalMatch = { - chunkId: string; - similarity: number; - textRank: number; - hybridScore: number; - reason: string; - fieldType?: string | null; - tableFacts?: Array<{ - id: string; - document_id: string; - source_chunk_id: string | null; - source_image_id: string | null; - page_number: number | null; - table_title: string | null; - row_label: string | null; - clinical_parameter: string | null; - threshold_value: string | null; - action: string | null; - text_rank?: number | null; - match_reason?: string | null; - }>; - indexUnit?: DocumentIndexUnitMatch | null; -}; - -type IndexUnitRpcRow = DocumentIndexUnitMatch & { - document_id: string; - source_chunk_id: string | null; - similarity?: number | null; - text_rank?: number | null; - hybrid_score?: number | null; -}; - -type TableFactRpcRow = { - id: string; - document_id: string; - source_chunk_id: string | null; - source_image_id: string | null; - page_number: number | null; - table_title: string | null; - row_label: string | null; - clinical_parameter: string | null; - threshold_value: string | null; - action: string | null; - text_rank?: number | null; - match_reason?: string | null; - metadata?: Record | null; -}; - -/** Document lookup chunk terms. */ -function documentLookupChunkTerms(query: string) { - const shortClinicalTerms = new Set(["ed", "im", "po", "pt"]); - return normalizedClinicalSearchTokens(query) - .filter((term) => term.length >= 3 || shortClinicalTerms.has(term)) - .slice(0, 8); -} - -/** Document lookup chunk score. */ -function documentLookupChunkScore(chunk: DocumentLookupChunkRow, terms: string[]) { - if (terms.length === 0) return 0; - const heading = chunk.section_heading?.toLowerCase() ?? ""; - const content = `${chunk.retrieval_synopsis ?? ""} ${chunk.content}`.toLowerCase(); - const matched = terms.filter((term) => heading.includes(term) || content.includes(term)); - const coverage = matched.length / terms.length; - const headingHits = matched.filter((term) => heading.includes(term)).length; - return coverage + headingHits * 0.08 + Math.max(0, 0.08 - chunk.chunk_index * 0.0005); -} - -/** Fetch best document lookup chunks. */ -async function fetchBestDocumentLookupChunks(args: { - supabase: ReturnType; - documentIds: string[]; - query: string; - limit: number; - ownerId?: string; - accessScope?: RetrievalAccessScope; - allowGlobalSearch?: boolean; -}) { - const terms = documentLookupChunkTerms(args.query); - const { data: rpcChunks, error: rpcError } = await callVersionedRetrievalRpc( - args.supabase, - "match_document_lookup_chunks_text_v2", - "match_document_lookup_chunks_text", - { - query_text: args.query, - document_filters: args.documentIds ?? undefined, - match_count: Math.max(args.limit * 3, 24), - ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), - }, - ); - if (!rpcError && rpcChunks?.length) { - const ranked = (rpcChunks as DocumentLookupChunkRow[]) - .map((chunk) => ({ - chunk, - score: Math.max(Number(chunk.text_rank ?? 0), documentLookupChunkScore(chunk, terms)), - })) - .filter((item) => item.score > 0) - .sort((a, b) => b.score - a.score || a.chunk.chunk_index - b.chunk.chunk_index) - .map((item) => item.chunk); - if (ranked.length) return { chunks: ranked.slice(0, args.limit), terms }; - } - - const safeFilters = terms - .map((term) => term.replace(/[%_,]/g, " ").trim()) - .filter(Boolean) - .flatMap((term) => [ - `content.ilike.%${term}%`, - `retrieval_synopsis.ilike.%${term}%`, - `section_heading.ilike.%${term}%`, - ]) - .join(","); - const baseQuery = args.supabase - .from("document_chunks") - .select( - "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids", - ) - .in("document_id", args.documentIds) - .limit(Math.max(args.limit * 4, 24)); - - const { data: matchedChunks, error: matchedError } = safeFilters - ? await baseQuery.or(safeFilters) - : await baseQuery.order("chunk_index", { ascending: true }); - - if (!matchedError && matchedChunks?.length) { - const ranked = (matchedChunks as DocumentLookupChunkRow[]) - .map((chunk) => ({ chunk, score: documentLookupChunkScore(chunk, terms) })) - .filter((item) => item.score > 0) - .sort((a, b) => b.score - a.score || a.chunk.chunk_index - b.chunk.chunk_index) - .map((item) => item.chunk); - if (ranked.length) return { chunks: ranked.slice(0, args.limit), terms }; - } - - const fallbackQuery = args.supabase - .from("document_chunks") - .select( - "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids", - ) - .in("document_id", args.documentIds) - .order("chunk_index", { ascending: true }) - .limit(args.limit); - const { data: fallbackChunks, error: fallbackError } = await fallbackQuery; - if (fallbackError || !fallbackChunks?.length) return { chunks: [] as DocumentLookupChunkRow[], terms }; - return { chunks: fallbackChunks as DocumentLookupChunkRow[], terms }; -} - -/** Fetch document title alias rows. */ -async function fetchDocumentTitleAliasRows(args: { - supabase: ReturnType; - query: string; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; -}) { - const terms = analyzeClinicalQuery(args.query) - .documentTitleTerms.map((term) => term.replace(/[%_,]/g, " ").replace(/\s+/g, " ").trim()) - .filter((term) => term.length >= 4) - .slice(0, 8); - if (!terms.length) return [] as DocumentLookupRow[]; - - const filters = terms.flatMap((term) => [`title.ilike.%${term}%`, `file_name.ilike.%${term}%`]).join(","); - let query = args.supabase - .from("documents") - .select("id,owner_id,title,file_name,status,page_count,chunk_count,image_count,metadata"); - if (typeof (query as { or?: unknown }).or !== "function") return [] as DocumentLookupRow[]; - query = query.or(filters).eq("status", "indexed").limit(12); - const accessScope = retrievalAccessScopeForArgs(args); - if (accessScope.ownerId && accessScope.includePublic) { - query = query.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); - } else if (accessScope.ownerId) { - query = query.eq("owner_id", accessScope.ownerId); - } else { - query = query.is("owner_id", null); - } - if (args.documentIds?.length) query = query.in("id", args.documentIds); - - const { data, error } = await query; - if (error || !data?.length) return [] as DocumentLookupRow[]; - - return (data as DocumentLookupRow[]).map((document) => ({ - ...document, - text_rank: Math.max(Number(document.text_rank ?? 0), 0.34), - match_reason: document.match_reason ?? "title_alias", - })); -} - -/** - * Retrieves and ranks document chunks for document-focused queries. - * - * @param args - Search configuration, including the required owner scope and optional document restrictions. - * @returns Ranked document lookup results, limited to `matchCount`. - */ -async function searchDocumentLookupFastPath(args: { - supabase: ReturnType; - query: string; - queryVariants?: string[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - matchCount: number; - telemetry?: SearchTelemetry; -}): Promise { - if (!args.ownerId) return [] as SearchResult[]; - const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( - 0, - maxTextRpcQueryVariants, - ); - const runDocumentLookup = async (variant: string, matchCount: number) => { - const { data, error } = await callVersionedRetrievalRpc( - args.supabase, - "match_documents_for_query_v2", - "match_documents_for_query", - { - query_text: variant, - match_count: matchCount, - ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), - }, - ); - if (error) recordHybridRpcError(args.telemetry, "match_documents_for_query", error); - if (error || !data?.length) return [] as DocumentLookupRow[]; - return data as DocumentLookupRow[]; - }; - const [primaryVariant, ...siblingVariants] = variants; - const primaryDocuments = primaryVariant === undefined ? [] : await runDocumentLookup(primaryVariant, 12); - const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryDocuments, 12); - const documentSets = skipSiblings - ? [primaryDocuments] - : [primaryDocuments, ...(await Promise.all(siblingVariants.map((variant) => runDocumentLookup(variant, 8))))]; - recordTextVariantFanout(args.telemetry, "match_documents_for_query", documentSets.length, skipSiblings); - const titleAliasDocuments = await fetchDocumentTitleAliasRows({ - supabase: args.supabase, - query: args.query, - ownerId: args.ownerId, - accessScope: args.accessScope, - documentIds: args.documentIds, - }); - const documentsById = new Map(); - for (const document of [...titleAliasDocuments, ...documentSets.flat()]) { - const existing = documentsById.get(document.id); - if (!existing || Number(document.text_rank ?? 0) > Number(existing.text_rank ?? 0)) { - documentsById.set(document.id, document); - } - } - const documents = Array.from(documentsById.values()); - if (!documents.length) return []; - - const allowedDocuments = args.documentIds?.length ? new Set(args.documentIds) : null; - const rankedDocuments = (documents as DocumentLookupRow[]) - .filter((document) => !allowedDocuments || allowedDocuments.has(document.id)) - .map((document) => ({ - document, - score: Math.min(0.34, Number(document.text_rank ?? 0)), - })) - .filter((item) => item.score > 0) - .sort((a, b) => b.score - a.score) - .slice(0, 5); - - if (rankedDocuments.length === 0) return []; - - const documentById = new Map(rankedDocuments.map((item) => [item.document.id, item.document])); - const scoreByDocument = new Map(rankedDocuments.map((item) => [item.document.id, item.score])); - const { chunks, terms } = await fetchBestDocumentLookupChunks({ - supabase: args.supabase, - documentIds: rankedDocuments.map((item) => item.document.id), - query: args.query, - limit: Math.max(args.matchCount, rankedDocuments.length * 4), - ownerId: args.ownerId, - accessScope: args.accessScope, - }); - - if (!chunks.length) return []; - - const results: SearchResult[] = []; - for (const chunk of chunks) { - const document = documentById.get(chunk.document_id); - if (!document) continue; - const documentScore = scoreByDocument.get(chunk.document_id) ?? 0; - const chunkScore = documentLookupChunkScore(chunk, terms); - // Not a cosine: fabricated from title/label match strength (RC9 — tagged below). - const similarity = Math.min(0.92, 0.58 + documentScore + Math.min(0.12, chunkScore * 0.08)); - results.push({ - similarity_origin: "synthetic_text", - id: chunk.id, - document_id: chunk.document_id, - title: document.title, - file_name: document.file_name, - page_number: chunk.page_number, - chunk_index: chunk.chunk_index, - section_heading: chunk.section_heading, - section_path: chunk.section_path ?? [], - heading_level: chunk.heading_level ?? null, - parent_heading: chunk.parent_heading ?? null, - anchor_id: chunk.anchor_id ?? null, - content: chunk.content, - retrieval_synopsis: chunk.retrieval_synopsis ?? null, - image_ids: chunk.image_ids ?? [], - source_metadata: normalizeSourceMetadata(document.metadata), - similarity, - text_rank: documentScore, - hybrid_score: Math.min(0.94, similarity + 0.02), - images: [], - }); - } - - return results - .sort( - (a, b) => (b.hybrid_score ?? b.similarity) - (a.hybrid_score ?? a.similarity) || a.chunk_index - b.chunk_index, - ) - .slice(0, args.matchCount); -} - /** Score explanation log metadata. */ function scoreExplanationLogMetadata(scoreExplanations: NonNullable) { return { @@ -2050,453 +1483,6 @@ function scoreExplanationLogMetadata(scoreExplanations: NonNullable 0) return Math.min(1, hybridScore); - return Math.min(1, card.confidence ?? 0.5); -} - -/** Load chunks for memory cards. */ -export async function loadChunksForMemoryCards( - supabase: ReturnType, - cards: DocumentMemoryCard[], - accessScope: RetrievalAccessScope, -) { - const documentIds = Array.from(new Set(cards.map((card) => card.document_id))).slice(0, 80); - if (documentIds.length === 0) return [] as SearchResult[]; - let documentQuery = supabase - .from("documents") - .select("id,title,file_name,metadata,owner_id,status") - .in("id", documentIds) - .eq("status", "indexed"); - if (accessScope.ownerId && accessScope.includePublic) { - documentQuery = documentQuery.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); - } else if (accessScope.ownerId) { - documentQuery = documentQuery.eq("owner_id", accessScope.ownerId); - } else { - documentQuery = documentQuery.is("owner_id", null); - } - const { data: documents, error: documentsError } = await documentQuery; - if (documentsError || !documents?.length) return [] as SearchResult[]; - - const documentById = new Map(documents.map((document) => [document.id, document])); - const allowedDocumentIds = new Set(documentById.keys()); - const chunkIds = Array.from( - new Set( - cards.filter((card) => allowedDocumentIds.has(card.document_id)).flatMap((card) => card.source_chunk_ids ?? []), - ), - ).slice(0, 80); - if (chunkIds.length === 0) return [] as SearchResult[]; - const { data: chunks, error: chunksError } = await supabase - .from("document_chunks") - .select( - "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids,index_generation_id", - ) - .in("id", chunkIds) - .in("document_id", [...allowedDocumentIds]) - .limit(chunkIds.length); - if (chunksError || !chunks?.length) return [] as SearchResult[]; - const bestCardByChunk = new Map(); - for (const card of cards) { - for (const chunkId of card.source_chunk_ids ?? []) { - const existing = bestCardByChunk.get(chunkId); - if (!existing || memoryCardChunkScore(card) > memoryCardChunkScore(existing)) bestCardByChunk.set(chunkId, card); - } - } - - return chunks - .map((chunk) => { - const document = documentById.get(chunk.document_id); - if (!document) return null; - const committedGeneration = committedIndexGeneration(document.metadata); - if (chunk.index_generation_id && chunk.index_generation_id !== committedGeneration) return null; - const card = bestCardByChunk.get(chunk.id); - // Not a cosine: fabricated from memory-card confidence (RC9 — tagged below). - const similarity = Math.min(0.92, 0.58 + (card?.confidence ?? 0.5) * 0.28); - return { - similarity_origin: "synthetic_text" as const, - id: chunk.id, - document_id: chunk.document_id, - title: document.title, - file_name: document.file_name, - page_number: chunk.page_number, - chunk_index: chunk.chunk_index, - section_heading: chunk.section_heading, - section_path: chunk.section_path ?? [], - heading_level: chunk.heading_level ?? null, - parent_heading: chunk.parent_heading ?? null, - anchor_id: chunk.anchor_id ?? null, - content: chunk.content, - retrieval_synopsis: chunk.retrieval_synopsis ?? null, - image_ids: chunk.image_ids ?? [], - source_metadata: normalizeSourceMetadata(document.metadata), - similarity, - text_rank: card?.confidence ?? 0, - hybrid_score: Math.min(0.96, similarity + 0.03), - images: [], - } satisfies SearchResult; - }) - .filter(Boolean) as SearchResult[]; -} - -/** Load chunks for signal matches. */ -export async function loadChunksForSignalMatches(args: { - supabase: ReturnType; - matches: ChunkSignalMatch[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; -}) { - const bestMatchByChunk = new Map(); - for (const match of args.matches) { - const existing = bestMatchByChunk.get(match.chunkId); - if (!existing || match.hybridScore > existing.hybridScore) bestMatchByChunk.set(match.chunkId, match); - } - const chunkIds = Array.from(bestMatchByChunk.keys()).slice(0, 80); - if (chunkIds.length === 0) return [] as SearchResult[]; - - const accessScope = retrievalAccessScopeForArgs(args); - const { data: chunkScopes, error: chunkScopesError } = await args.supabase - .from("document_chunks") - .select("id,document_id") - .in("id", chunkIds) - .limit(chunkIds.length); - if (chunkScopesError || !chunkScopes?.length) return [] as SearchResult[]; - - const documentIds = Array.from(new Set(chunkScopes.map((chunk) => chunk.document_id))); - let documentQuery = args.supabase - .from("documents") - .select("id,title,file_name,metadata,owner_id,status") - .in("id", documentIds) - .eq("status", "indexed"); - if (accessScope.ownerId && accessScope.includePublic) { - documentQuery = documentQuery.or(`owner_id.eq.${accessScope.ownerId},owner_id.is.null`); - } else if (accessScope.ownerId) { - documentQuery = documentQuery.eq("owner_id", accessScope.ownerId); - } else { - documentQuery = documentQuery.is("owner_id", null); - } - const { data: documents, error: documentsError } = await documentQuery; - if (documentsError || !documents?.length) return [] as SearchResult[]; - const documentById = new Map(documents.map((document) => [document.id, document])); - const allowedDocumentIds = new Set(documentById.keys()); - const allowedChunkIds = chunkScopes - .filter((chunk) => allowedDocumentIds.has(chunk.document_id)) - .map((chunk) => chunk.id); - if (allowedChunkIds.length === 0) return [] as SearchResult[]; - const { data: chunks, error: chunksError } = await args.supabase - .from("document_chunks") - .select( - "id,document_id,page_number,chunk_index,section_heading,section_path,heading_level,parent_heading,anchor_id,content,retrieval_synopsis,image_ids,index_generation_id", - ) - .in("id", allowedChunkIds) - .in("document_id", [...allowedDocumentIds]) - .limit(allowedChunkIds.length); - if (chunksError || !chunks?.length) return [] as SearchResult[]; - - return chunks - .map((chunk) => { - const document = documentById.get(chunk.document_id); - const match = bestMatchByChunk.get(chunk.id); - if (!document || !match) return null; - const committedGeneration = committedIndexGeneration(document.metadata); - if (chunk.index_generation_id && chunk.index_generation_id !== committedGeneration) return null; - return { - id: chunk.id, - document_id: chunk.document_id, - title: document.title, - file_name: document.file_name, - page_number: chunk.page_number, - chunk_index: chunk.chunk_index, - section_heading: chunk.section_heading, - section_path: chunk.section_path ?? [], - heading_level: chunk.heading_level ?? null, - parent_heading: chunk.parent_heading ?? null, - anchor_id: chunk.anchor_id ?? null, - content: chunk.content, - retrieval_synopsis: chunk.retrieval_synopsis ?? null, - image_ids: chunk.image_ids ?? [], - source_metadata: normalizeSourceMetadata(document.metadata), - // ChunkSignalMatch similarities are fabricated from table-fact text rank (RC9). - similarity_origin: "synthetic_text" as const, - similarity: match.similarity, - text_rank: match.textRank, - hybrid_score: match.hybridScore, - table_facts: match.tableFacts?.filter((fact) => - isReviewedTablePromotable((fact as { metadata?: unknown }).metadata), - ), - index_unit: match.indexUnit ?? null, - match_explanation: { - sectionHit: match.reason === "section_context" || match.indexUnit?.unit_type === "section_summary", - tableHit: match.reason.startsWith("table") || match.indexUnit?.unit_type === "table_fact", - indexUnitType: match.indexUnit?.unit_type ?? null, - vectorSimilarity: match.similarity, - textRank: match.textRank, - fieldType: match.fieldType ?? null, - freshness: normalizeSourceMetadata(document.metadata).document_status, - extractionQuality: normalizeSourceMetadata(document.metadata).extraction_quality, - reasons: [match.reason, match.indexUnit?.unit_type ? `unit:${match.indexUnit.unit_type}` : ""].filter( - Boolean, - ), - }, - images: [], - } satisfies SearchResult; - }) - .filter(Boolean) as SearchResult[]; -} - -/** - * Retrieves document chunks containing table facts relevant to a query. - * - * @param args - Retrieval options, including query variants, scope filters, result limit, and optional telemetry. - * @returns Search results containing the highest-ranked table facts grouped by source chunk. - */ -async function searchTableFactCandidates(args: { - supabase: ReturnType; - query: string; - queryVariants?: string[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - allowGlobalSearch?: boolean; - matchCount: number; - telemetry?: SearchTelemetry; -}) { - const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( - 0, - maxTextRpcQueryVariants, - ); - const runTableFacts = async (variant: string, matchCount: number) => { - const { data, error } = await callVersionedRetrievalRpc( - args.supabase, - "match_document_table_facts_text_v2", - "match_document_table_facts_text", - { - query_text: variant, - match_count: matchCount, - document_filters: args.documentIds ?? undefined, - ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), - }, - ); - if (error) recordHybridRpcError(args.telemetry, "match_document_table_facts_text", error); - if (error || !data?.length) return [] as TableFactRpcRow[]; - return data as TableFactRpcRow[]; - }; - const [primaryVariant, ...siblingVariants] = variants; - const primaryFacts = primaryVariant === undefined ? [] : await runTableFacts(primaryVariant, args.matchCount); - const skipSiblings = siblingVariants.length > 0 && firstVariantPoolIsStrong(primaryFacts, args.matchCount); - const factSets = skipSiblings - ? [primaryFacts] - : [ - primaryFacts, - ...(await Promise.all(siblingVariants.map((variant) => runTableFacts(variant, Math.min(args.matchCount, 24))))), - ]; - recordTextVariantFanout(args.telemetry, "match_document_table_facts_text", factSets.length, skipSiblings); - const data = factSets.flat(); - if (!data.length) return [] as SearchResult[]; - - const grouped = new Map(); - for (const fact of data) { - if (!fact.source_chunk_id) continue; - const textRank = Number(fact.text_rank ?? 0); - const current = grouped.get(fact.source_chunk_id); - const tableFact = { ...fact, text_rank: textRank, match_reason: fact.match_reason ?? "table_row" }; - const next: ChunkSignalMatch = { - chunkId: fact.source_chunk_id, - similarity: Math.min(0.94, 0.62 + Math.min(textRank, 1) * 0.3), - textRank, - hybridScore: Math.min(0.97, 0.66 + Math.min(textRank, 1) * 0.3), - reason: fact.match_reason ?? "table_row", - tableFacts: [...(current?.tableFacts ?? []), tableFact].slice(0, 5), - }; - if (!current || next.hybridScore > current.hybridScore) grouped.set(fact.source_chunk_id, next); - } - - return loadChunksForSignalMatches({ - supabase: args.supabase, - matches: Array.from(grouped.values()), - ownerId: args.ownerId, - accessScope: args.accessScope, - }); -} - -/** Search embedding field candidates. */ -async function searchEmbeddingFieldCandidates(args: { - supabase: ReturnType; - query: string; - queryEmbedding: number[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - allowGlobalSearch?: boolean; - matchCount: number; - telemetry?: SearchTelemetry; -}) { - const { data, error } = await callVersionedRetrievalRpc( - args.supabase, - "match_document_embedding_fields_hybrid_v2", - "match_document_embedding_fields_hybrid", - { - query_embedding: args.queryEmbedding as unknown as string, - query_text: buildClinicalTextSearchQuery(args.query), - match_count: args.matchCount, - min_similarity: 0.12, - document_filters: args.documentIds ?? undefined, - ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), - }, - ); - if (error) recordHybridRpcError(args.telemetry, "match_document_embedding_fields_hybrid", error); - if (error || !data?.length) return [] as SearchResult[]; - const matches = ( - data as Array<{ - source_chunk_id: string | null; - field_type: string | null; - similarity?: number | null; - text_rank?: number | null; - hybrid_score?: number | null; - }> - ) - .filter( - ( - row, - ): row is { - source_chunk_id: string; - field_type: string | null; - similarity?: number | null; - text_rank?: number | null; - hybrid_score?: number | null; - } => Boolean(row.source_chunk_id), - ) - .map((row) => ({ - chunkId: row.source_chunk_id, - similarity: Number(row.similarity ?? 0), - textRank: Number(row.text_rank ?? 0), - hybridScore: Number(row.hybrid_score ?? row.similarity ?? 0), - reason: "section_context", - fieldType: row.field_type, - })); - return loadChunksForSignalMatches({ - supabase: args.supabase, - matches, - ownerId: args.ownerId, - accessScope: args.accessScope, - }); -} - -/** Search index unit candidates. */ -async function searchIndexUnitCandidates(args: { - supabase: ReturnType; - query: string; - queryEmbedding: number[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - allowGlobalSearch?: boolean; - matchCount: number; - telemetry?: SearchTelemetry; -}) { - const { data, error } = await callVersionedRetrievalRpc( - args.supabase, - "match_document_index_units_hybrid_v2", - "match_document_index_units_hybrid", - { - query_embedding: args.queryEmbedding as unknown as string, - query_text: buildClinicalTextSearchQuery(args.query), - match_count: args.matchCount, - min_similarity: 0.1, - document_filters: args.documentIds ?? undefined, - ...retrievalRpcScopeArgs(retrievalAccessScopeForArgs(args)), - }, - ); - if (error) recordHybridRpcError(args.telemetry, "match_document_index_units_hybrid", error); - if (error || !data?.length) return [] as SearchResult[]; - const matches = (data as IndexUnitRpcRow[]) - .filter((row): row is IndexUnitRpcRow & { source_chunk_id: string } => Boolean(row.source_chunk_id)) - .map((row) => ({ - chunkId: row.source_chunk_id, - similarity: Number(row.similarity ?? 0), - textRank: Number(row.text_rank ?? 0), - hybridScore: Number(row.hybrid_score ?? row.similarity ?? 0), - reason: `index_unit:${row.unit_type}`, - fieldType: row.unit_type, - indexUnit: { - id: row.id, - document_id: row.document_id, - unit_type: row.unit_type, - title: row.title, - content: row.content, - source_chunk_id: row.source_chunk_id, - source_image_id: row.source_image_id, - page_start: row.page_start, - page_end: row.page_end, - heading_path: row.heading_path ?? [], - normalized_terms: row.normalized_terms ?? [], - source_span: row.source_span ?? null, - quality_score: row.quality_score, - extraction_mode: row.extraction_mode, - similarity: row.similarity, - text_rank: row.text_rank, - hybrid_score: row.hybrid_score, - metadata: row.metadata ?? null, - }, - })); - return loadChunksForSignalMatches({ - supabase: args.supabase, - matches, - ownerId: args.ownerId, - accessScope: args.accessScope, - }); -} - -type MemoryCardCache = Map>; - -/** With memory boosted candidates. */ -async function withMemoryBoostedCandidates(args: { - supabase: ReturnType; - query: string; - candidates: SearchResult[]; - queryEmbedding?: number[]; - ownerId?: string; - accessScope?: RetrievalAccessScope; - documentIds?: string[]; - matchCount: number; - cardCache?: MemoryCardCache; -}) { - // A3: the memory-card fetch is invoked at several waterfall stages. Memoize per request, - // scoped by owner/document filters because fetchMemoryCardsForQuery applies those filters. - const effectiveMatchCount = Math.max(args.matchCount, 48); - const documentScope = args.documentIds?.length ? [...args.documentIds].sort().join(",") : "all-documents"; - const cacheKey = [ - retrievalAccessScopeKey(retrievalAccessScopeForArgs(args)), - documentScope, - args.query, - args.queryEmbedding?.length ? "vec" : "text", - effectiveMatchCount, - ].join("\0"); - let cardsPromise = args.cardCache?.get(cacheKey); - if (!cardsPromise) { - cardsPromise = fetchMemoryCardsForQuery({ - supabase: args.supabase, - query: args.query, - queryEmbedding: args.queryEmbedding, - ownerId: args.ownerId, - accessScope: args.accessScope, - documentIds: args.documentIds, - matchCount: effectiveMatchCount, - }); - args.cardCache?.set(cacheKey, cardsPromise); - } - const cards = await cardsPromise; - if (cards.length === 0) return { results: args.candidates, cards }; - - const memoryChunkResults = await loadChunksForMemoryCards(args.supabase, cards, retrievalAccessScopeForArgs(args)); - const merged = mergeSearchResults(memoryChunkResults, args.candidates); - return { - results: applyMemoryCardBoosts(args.query, merged, cards), - cards, - }; -} - type DocumentRankingMetadataCache = { documentMetadata: Map< string,