diff --git a/.env.example b/.env.example
index d5e826e49..dadd4d6ae 100644
--- a/.env.example
+++ b/.env.example
@@ -7,6 +7,17 @@ SUPABASE_PROJECT_NAME=Clinical KB Database
NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=your-publishable-or-anon-key
SUPABASE_SERVICE_ROLE_KEY=your-service-role-key
+# Local-only no-auth mode (development only).
+# Enable one or both of these to load real data without per-request browser auth:
+# - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server)
+# - LOCAL_NO_AUTH (server only)
+#NEXT_PUBLIC_LOCAL_NO_AUTH=false
+#LOCAL_NO_AUTH=false
+
+# Optional local no-auth owner identity. Set one of these in local dev.
+#LOCAL_NO_AUTH_OWNER_ID=
+#LOCAL_NO_AUTH_OWNER_EMAIL=
+
# OpenAI direct API. This app sends extracted guideline text and extracted images
# to OpenAI for embeddings, captioning, and grounded answer generation.
OPENAI_API_KEY=replace-with-openai-api-key
@@ -14,7 +25,7 @@ OPENAI_EMBEDDING_MODEL=text-embedding-3-small
OPENAI_ANSWER_MODEL=gpt-5.4-mini
OPENAI_FAST_ANSWER_MODEL=gpt-5.4-mini
OPENAI_STRONG_ANSWER_MODEL=gpt-5.4
-OPENAI_MAX_OUTPUT_TOKENS=900
+OPENAI_MAX_OUTPUT_TOKENS=1400
OPENAI_QUERY_CACHE_SIZE=200
OPENAI_VISION_MODEL=gpt-5.4-mini
OPENAI_REQUEST_TIMEOUT_MS=45000
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 58ec3f823..efa46cf57 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,10 +13,10 @@ jobs:
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: Setup Node.js
- uses: actions/setup-node@v4
+ uses: actions/setup-node@v6
with:
node-version: 22
cache: npm
diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml
index 42a422724..367b11810 100644
--- a/.github/workflows/secret-scan.yml
+++ b/.github/workflows/secret-scan.yml
@@ -19,11 +19,11 @@ jobs:
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Scan for secrets
- uses: gitleaks/gitleaks-action@v2
+ uses: gitleaks/gitleaks-action@v3
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/AGENTS.md b/AGENTS.md
index 1bd3f6f39..bd2f576bf 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -27,3 +27,103 @@ This version has breaking changes — APIs, conventions, and file structure may
- Older unused project ref `qjgitjyhxrwxsrydablr` belongs to `Database`; treat it as stale and do not use it.
- Run `npm run check:supabase-project` after changing Supabase env values.
+
+
+
+# `upload` shortcut
+
+When the user types exactly:
+
+upload
+
+as the entire task message, treat it as a shortcut for the safe Git handoff workflow below.
+
+The goal is to leave useful completed work safely committed and, where safe, pushed to the current feature branch. The goal is not to merge into main, delete branches, discard work, force-push, close PRs, deploy, or perform destructive cleanup without explicit user confirmation.
+
+## Protected and base branches
+
+Treat `main`, `master`, `develop`, and `release/*` as protected/base branches for this workflow.
+
+If `upload` is run while on `main`, automatically create or use a branch named exactly `temporary` before staging, committing, or pushing, then continue the upload workflow from `temporary`:
+
+- If neither local `temporary` nor `origin/temporary` exists, run `git switch -c temporary`.
+- If local `temporary` exists and is not checked out in another worktree, switch to it only when it is clearly safe.
+- If `origin/temporary` exists, use it only when it is clearly the matching intended branch.
+- If any `temporary` branch state is ambiguous, diverged, checked out elsewhere, or unsafe, stop and ask instead of overwriting.
+
+If already on a non-protected feature branch, continue using that branch.
+
+## Required inspection
+
+Start with read-only inspection before making changes. Check:
+
+- Current branch or detached HEAD state
+- `git status`
+- Staged, unstaged, and untracked files
+- Recent commits relevant to the current branch
+- Remote configuration and upstream branch
+- Whether the branch is ahead, behind, or diverged
+- Whether the current branch appears protected/base
+- Other Git worktrees, if detectable
+- Available checks such as tests, lint, type check, or build scripts
+- Existing branch, commit, PR, and release-flow conventions
+
+Do not assume branch names, remotes, package managers, test commands, deployment targets, or project structure. Inspect first.
+
+## Safe actions allowed without further confirmation
+
+When the repository state makes it clearly safe, you may:
+
+- Stage coherent completed changes that clearly belong together
+- Create one or more logical commits with clear messages based on the diff
+- Fast-forward pull only when there are no local commits or conflict risks
+- Push the current non-protected feature branch if it has a valid upstream
+- Set an upstream for the current feature branch only when the correct remote and branch name are obvious
+- Leave the worktree clean by committing safe completed changes
+
+## Actions requiring explicit confirmation
+
+Do not perform these without asking the user first:
+
+- `git reset --hard`
+- `git clean -fd` or other destructive cleanup
+- Discarding, overwriting, or reverting uncommitted changes
+- Deleting local or remote branches
+- Renaming branches
+- Force-pushing
+- Rebasing a shared/public branch
+- Resolving divergent branch history
+- Merging into `main`, `master`, `develop`, `release/*`, or any protected/base branch
+- Closing pull requests
+- Changing GitHub default branch, branch protection, repository settings, or deployment settings
+- Modifying production data or deployment configuration
+- Committing secrets, credentials, tokens, private keys, or sensitive local configuration
+- Updating branch references where the correct replacement branch is ambiguous
+
+If any of these seem necessary, stop and report what is risky, why it is risky, the recommended next step, and the exact confirmation needed.
+
+## Mixed, suspicious, or unsafe changes
+
+Do not automatically commit files that look like `.env` files, credentials, secrets, logs, caches, build artifacts, editor or OS files, temporary/debug files, or generated files not normally committed by this project. Report only the path and concern for possible secrets; never print secret values.
+
+If changes appear unrelated, incomplete, experimental, or WIP, do not commit everything together automatically. Commit only clearly coherent completed changes when safe; otherwise summarize the groups and ask what should be included.
+
+## Branch cleanup and reference updates
+
+If stale, inappropriate, merged, or unnecessary branches are detected, list cleanup candidates but do not delete or rename branches automatically.
+
+Before recommending deletion or rename, audit accessible references including `.github/workflows/*`, CI/CD config, deployment config, scripts, package scripts, docs, release notes or release scripts, safe environment/config files, branch-specific config, open PR metadata if accessible, and GitHub branch protection/default branch metadata if safely accessible.
+
+Update repo-tracked references to a renamed or replacement branch only when the old branch reference is clearly found, the replacement is obvious, the change is low-risk, and the user has approved the branch rename or deletion. If the replacement is unclear, report the reference and ask what it should point to.
+
+## Syncing and verification
+
+Do not rebase, merge, or resolve remote divergence automatically. Fast-forward pulls are allowed only when clearly safe. Push only the current non-protected feature branch when clearly safe.
+
+Run the smallest relevant checks that are available and appropriate, such as tests, lint, type check, or build checks. Do not claim checks passed unless they were actually run. If checks cannot be run, explain why and state the command that would normally be used.
+
+## Final report
+
+After completing `upload`, summarize the current branch and worktree state, whether the worktree is clean, what changed, files committed, commit hash and message if created, whether anything was pushed, remote branch and likely PR target, checks run and results, checks not run and why, branch cleanup candidates, branch references found or updated, risky actions skipped, and exact confirmation needed for any recommended follow-up.
+
+
diff --git a/package.json b/package.json
index aefa51570..547b235f6 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,9 @@
"check:indexing": "tsx scripts/check-indexing.ts",
"eval:rag": "tsx scripts/eval-rag.ts",
"eval:search": "tsx scripts/eval-search.ts",
+ "cleanup:storage": "tsx scripts/cleanup-storage.ts",
+ "purge:query-logs": "tsx scripts/purge-query-logs.ts",
+ "audit:tables": "tsx scripts/audit-tables.ts",
"samples": "tsx scripts/generate-sample-documents.ts",
"samples:check": "tsx scripts/check-sample-extraction.ts"
},
diff --git a/scripts/audit-tables.ts b/scripts/audit-tables.ts
new file mode 100644
index 000000000..b6a20d883
--- /dev/null
+++ b/scripts/audit-tables.ts
@@ -0,0 +1,188 @@
+import { loadEnvConfig } from "@next/env";
+import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils";
+import { assessClinicalImageUse } from "@/lib/image-filtering";
+
+loadEnvConfig(process.cwd());
+
+type Args = {
+ ownerEmail?: string;
+ allOwners: boolean;
+ document?: string;
+ limit: number;
+ failOnMissed: boolean;
+};
+
+function parseArgs(): Args {
+ const args: Args = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, allOwners: !process.env.RAG_EVAL_OWNER_EMAIL, limit: 200, failOnMissed: false };
+ const tokens = process.argv.slice(2);
+ for (let index = 0; index < tokens.length; index += 1) {
+ const token = tokens[index];
+ const value = tokens[index + 1];
+ if (token === "--owner-email") {
+ args.ownerEmail = value;
+ args.allOwners = false;
+ index += 1;
+ } else if (token === "--all-owners") {
+ args.allOwners = true;
+ } else if (token === "--document") {
+ args.document = value;
+ index += 1;
+ } else if (token === "--limit") {
+ args.limit = Math.max(1, Math.min(Number(value) || 200, 1000));
+ index += 1;
+ } else if (token === "--fail-on-missed") {
+ args.failOnMissed = true;
+ }
+ }
+ return args;
+}
+
+function textSuggestsTable(text: string) {
+ return /\b(table\s+\d+[a-z]?|appendix\s+\d+[a-z]?|roles?\s+and\s+responsibilities|score\b.*\bmanagement|observation|medication|dose|frequency)\b/i.test(
+ text,
+ );
+}
+
+function compactTitle(title: string) {
+ return title.length > 72 ? `${title.slice(0, 69).trim()}...` : title;
+}
+
+async function main() {
+ const args = parseArgs();
+ if (!args.ownerEmail && !args.allOwners) throw new Error('Provide --owner-email "you@example.com" or --all-owners.');
+
+ const supabase = await loadAdminClient();
+ const ownerId = args.ownerEmail && !args.allOwners ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined;
+
+ let documentQuery = supabase
+ .from("documents")
+ .select("id,title,file_name,status,page_count,image_count")
+ .order("created_at", { ascending: true })
+ .limit(args.limit);
+ if (ownerId) documentQuery = documentQuery.eq("owner_id", ownerId);
+ if (args.document) {
+ documentQuery = documentQuery.or(
+ `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`,
+ );
+ }
+
+ const { data: documents, error: documentsError } = await documentQuery;
+ if (documentsError) throw new Error(documentsError.message);
+ const documentIds = (documents ?? []).map((document) => document.id as string);
+ if (documentIds.length === 0) {
+ console.log("No documents matched the audit filters.");
+ return;
+ }
+
+ const [imagesResult, pagesResult] = await Promise.all([
+ supabase
+ .from("document_images")
+ .select("document_id,source_kind,searchable,image_type,clinical_relevance_score,metadata")
+ .in("document_id", documentIds)
+ .neq("image_type", "logo_decorative"),
+ supabase.from("document_pages").select("document_id,page_number,text").in("document_id", documentIds),
+ ]);
+ if (imagesResult.error) throw new Error(imagesResult.error.message);
+ if (pagesResult.error) throw new Error(pagesResult.error.message);
+
+ const counts = new Map();
+ for (const documentId of documentIds) {
+ counts.set(documentId, { tables: 0, clinicalTables: 0, adminTables: 0, searchableAdminTables: 0, images: 0 });
+ }
+ for (const image of imagesResult.data ?? []) {
+ const documentId = String(image.document_id);
+ const current = counts.get(documentId) ?? {
+ tables: 0,
+ clinicalTables: 0,
+ adminTables: 0,
+ searchableAdminTables: 0,
+ images: 0,
+ };
+ const metadata = image.metadata && typeof image.metadata === "object" ? (image.metadata as Record) : {};
+ const useClass = String(
+ metadata.clinical_use_class ??
+ assessClinicalImageUse({
+ imageType: image.image_type,
+ searchable: image.searchable,
+ clinicalRelevanceScore: image.clinical_relevance_score,
+ sourceKind: image.source_kind,
+ tableRole: typeof metadata.table_role === "string" ? metadata.table_role : null,
+ tableText:
+ typeof metadata.table_text === "string"
+ ? metadata.table_text
+ : typeof metadata.table_text_snippet === "string"
+ ? metadata.table_text_snippet
+ : null,
+ }).clinical_use_class,
+ );
+ if (image.searchable !== false) current.images += 1;
+ if (image.source_kind === "table_crop") {
+ current.tables += 1;
+ if (useClass === "clinical_evidence" && image.searchable !== false) current.clinicalTables += 1;
+ if (["administrative", "reference"].includes(useClass)) current.adminTables += 1;
+ if (["administrative", "reference"].includes(useClass) && image.searchable !== false) {
+ current.searchableAdminTables += 1;
+ }
+ }
+ counts.set(documentId, current);
+ }
+
+ const tableMarkerPages = new Map>();
+ for (const page of pagesResult.data ?? []) {
+ if (!textSuggestsTable(String(page.text ?? ""))) continue;
+ const documentId = String(page.document_id);
+ const pages = tableMarkerPages.get(documentId) ?? new Set();
+ const pageNumber = Number(page.page_number);
+ if (Number.isFinite(pageNumber)) pages.add(pageNumber);
+ tableMarkerPages.set(documentId, pages);
+ }
+
+ const possibleMisses: string[] = [];
+ const searchableAdminIssues: string[] = [];
+ console.log(`Table audit for ${documents?.length ?? 0} documents`);
+ for (const document of documents ?? []) {
+ const documentCounts = counts.get(document.id) ?? {
+ tables: 0,
+ clinicalTables: 0,
+ adminTables: 0,
+ searchableAdminTables: 0,
+ images: 0,
+ };
+ const markerPages = Array.from(tableMarkerPages.get(document.id) ?? []).sort((a, b) => a - b);
+ if (markerPages.length > 0 && documentCounts.tables === 0) {
+ possibleMisses.push(`${document.file_name}: table-like text on pages ${markerPages.slice(0, 8).join(", ")}`);
+ }
+ if (documentCounts.searchableAdminTables > 0) {
+ searchableAdminIssues.push(`${document.file_name}: searchable admin/reference tables=${documentCounts.searchableAdminTables}`);
+ }
+ console.log(
+ [
+ compactTitle(document.file_name ?? document.title ?? document.id),
+ `status=${document.status}`,
+ `pages=${document.page_count ?? 0}`,
+ `tables=${documentCounts.tables}`,
+ `clinicalTables=${documentCounts.clinicalTables}`,
+ `adminReferenceTables=${documentCounts.adminTables}`,
+ `searchableAdminReference=${documentCounts.searchableAdminTables}`,
+ `searchableImages=${documentCounts.images}`,
+ markerPages.length ? `tableTextPages=${markerPages.slice(0, 8).join(",")}` : "tableTextPages=none",
+ ].join(" | "),
+ );
+ }
+
+ console.log(`Possible missed table documents: ${possibleMisses.length}`);
+ for (const item of possibleMisses.slice(0, 20)) console.log(`- ${item}`);
+ console.log(`Searchable admin/reference table issues: ${searchableAdminIssues.length}`);
+ for (const item of searchableAdminIssues.slice(0, 20)) console.log(`- ${item}`);
+ if (searchableAdminIssues.length > 0) {
+ throw new Error("Table audit found admin/reference tables still marked searchable.");
+ }
+ if (args.failOnMissed && possibleMisses.length > 0) {
+ throw new Error("Table audit found documents with table-like text but no retained table crops.");
+ }
+}
+
+main().catch((error) => {
+ console.error(error instanceof Error ? error.message : error);
+ process.exit(1);
+});
diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts
index 7efdeb6ee..cdd7c2d4c 100644
--- a/scripts/check-indexing.ts
+++ b/scripts/check-indexing.ts
@@ -2,14 +2,108 @@ import { loadEnvConfig } from "@next/env";
loadEnvConfig(process.cwd());
-async function main() {
- const [{ env, requireOpenAIEnv, requireServerEnv }, { createAdminClient }, { checkPythonPdfPrerequisites }] =
- await Promise.all([
- import("@/lib/env"),
- import("@/lib/supabase/admin"),
- import("../worker/prerequisites"),
+type MetadataRow = { document_id: string; metadata?: unknown; source?: string | null };
+type QueryResponse = {
+ data: T[] | null;
+ error: { message: string } | null;
+ count?: number | null;
+};
+type QueryBuilder = PromiseLike> & {
+ eq(column: string, value: unknown): QueryBuilder;
+ gt(column: string, value: unknown): QueryBuilder;
+ in(column: string, values: unknown[]): QueryBuilder;
+ is(column: string, value: unknown): QueryBuilder;
+ order(column: string, options?: Record): QueryBuilder;
+ range(from: number, to: number): PromiseLike>;
+ limit(count: number): PromiseLike>;
+};
+type SupabaseLike = {
+ from(table: string): {
+ select(columns: string, options?: Record): QueryBuilder;
+ };
+};
+
+function metadataRecord(metadata: unknown): Record {
+ return metadata && typeof metadata === "object" && !Array.isArray(metadata)
+ ? (metadata as Record)
+ : {};
+}
+
+function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) {
+ return metadataRecord(metadata).rag_enrichment_version === expectedVersion;
+}
+
+function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) {
+ const record = metadataRecord(metadata);
+ return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion;
+}
+
+function strictEnrichmentVersionRequired() {
+ return process.argv.includes("--strict-enrichment-version") || process.env.RAG_REQUIRE_CURRENT_ENRICHMENT_VERSION === "1";
+}
+
+async function loadEnrichmentRows(supabase: SupabaseLike, documentIds: string[]) {
+ const summaries: MetadataRow[] = [];
+ const labels: MetadataRow[] = [];
+
+ for (let start = 0; start < documentIds.length; start += 100) {
+ const ids = documentIds.slice(start, start + 100);
+ const [summaryResult, labelResult] = await Promise.all([
+ supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids),
+ supabase.from("document_labels").select("document_id,source,metadata").in("document_id", ids),
]);
+ if (summaryResult.error) throw new Error(summaryResult.error.message);
+ if (labelResult.error) throw new Error(labelResult.error.message);
+ summaries.push(...((summaryResult.data ?? []) as MetadataRow[]));
+ labels.push(...((labelResult.data ?? []) as MetadataRow[]));
+ }
+
+ return { summaries, labels };
+}
+
+async function loadRowsForDocuments(supabase: SupabaseLike, table: string, select: string, documentIds: string[]) {
+ const rows: MetadataRow[] = [];
+ for (let start = 0; start < documentIds.length; start += 100) {
+ const ids = documentIds.slice(start, start + 100);
+ for (let rangeStart = 0; ; rangeStart += 1000) {
+ const { data, error } = await supabase
+ .from(table)
+ .select(select)
+ .in("document_id", ids)
+ .range(rangeStart, rangeStart + 999);
+ if (error) throw new Error(error.message);
+ rows.push(...((data ?? []) as MetadataRow[]));
+ if (!data || data.length < 1000) break;
+ }
+ }
+ return rows;
+}
+
+async function loadDeepMemoryRows(supabase: SupabaseLike, documentIds: string[]) {
+ const [sections, memoryCards, chunks] = await Promise.all([
+ loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds),
+ loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds),
+ loadRowsForDocuments(supabase, "document_chunks", "document_id,metadata", documentIds),
+ ]);
+ return { sections, memoryCards, chunks };
+}
+
+async function main() {
+ const [
+ { env, requireOpenAIEnv, requireServerEnv },
+ { createAdminClient },
+ { checkPythonPdfPrerequisites },
+ { ragEnrichmentVersion },
+ { ragDeepMemoryVersion },
+ ] = await Promise.all([
+ import("@/lib/env"),
+ import("@/lib/supabase/admin"),
+ import("../worker/prerequisites"),
+ import("@/lib/document-enrichment"),
+ import("@/lib/deep-memory"),
+ ]);
+
requireServerEnv();
requireOpenAIEnv();
@@ -23,11 +117,188 @@ async function main() {
if (batchError) throw new Error(batchError.message);
const { error: jobError } = await supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts").limit(1);
if (jobError) throw new Error(jobError.message);
+ const { error: cleanupTableError } = await supabase.from("storage_cleanup_jobs").select("id,status").limit(1);
+ if (cleanupTableError) throw new Error(cleanupTableError.message);
+
+ const { data: documents, error: documentsError } = await supabase
+ .from("documents")
+ .select("id,owner_id,title,content_hash,status,page_count,chunk_count,metadata");
+ if (documentsError) throw new Error(documentsError.message);
+
+ const supabaseForChecks = supabase as unknown as SupabaseLike;
+ const indexedDocuments = (documents ?? []).filter((document) => document.status === "indexed");
+ const enrichmentRows = await loadEnrichmentRows(
+ supabaseForChecks,
+ indexedDocuments.map((document) => document.id),
+ );
+ const deepMemoryRows = await loadDeepMemoryRows(
+ supabaseForChecks,
+ indexedDocuments.map((document) => document.id),
+ );
+ const summariesByDocument = new Map(enrichmentRows.summaries.map((row) => [row.document_id, row]));
+ const labelRowsByDocument = new Map();
+ const sectionRowsByDocument = new Map();
+ const memoryRowsByDocument = new Map();
+ const chunkRowsByDocument = new Map();
+ for (const label of enrichmentRows.labels) {
+ labelRowsByDocument.set(label.document_id, [...(labelRowsByDocument.get(label.document_id) ?? []), label]);
+ }
+ for (const section of deepMemoryRows.sections) {
+ sectionRowsByDocument.set(section.document_id, [...(sectionRowsByDocument.get(section.document_id) ?? []), section]);
+ }
+ for (const card of deepMemoryRows.memoryCards) {
+ memoryRowsByDocument.set(card.document_id, [...(memoryRowsByDocument.get(card.document_id) ?? []), card]);
+ }
+ for (const chunk of deepMemoryRows.chunks) {
+ chunkRowsByDocument.set(chunk.document_id, [...(chunkRowsByDocument.get(chunk.document_id) ?? []), chunk]);
+ }
+
+ const duplicateHashGroups = new Map();
+ for (const document of documents ?? []) {
+ if (!document.owner_id || !document.content_hash) continue;
+ const key = `${document.owner_id}:${document.content_hash}`;
+ duplicateHashGroups.set(key, [...(duplicateHashGroups.get(key) ?? []), document.title ?? document.id]);
+ }
+ const duplicateGroups = Array.from(duplicateHashGroups.values()).filter((titles) => titles.length > 1);
+ const emptyIndexedDocuments = (documents ?? []).filter(
+ (document) =>
+ document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0),
+ );
+ const documentsWithChunkCountMismatch = indexedDocuments.filter(
+ (document) => (chunkRowsByDocument.get(document.id) ?? []).length !== (document.chunk_count ?? 0),
+ );
+ const documentsMissingSummaries = indexedDocuments.filter((document) => !summariesByDocument.has(document.id));
+ const documentsMissingGeneratedLabels = indexedDocuments.filter((document) =>
+ (labelRowsByDocument.get(document.id) ?? []).every((label) => label.source !== "generated"),
+ );
+ const documentsWithCurrentEnrichmentVersion = indexedDocuments.filter((document) => {
+ const summary = summariesByDocument.get(document.id);
+ const generatedLabels = (labelRowsByDocument.get(document.id) ?? []).filter((label) => label.source === "generated");
+ return (
+ hasCurrentEnrichmentVersion(document.metadata, ragEnrichmentVersion) &&
+ hasCurrentEnrichmentVersion(summary?.metadata, ragEnrichmentVersion) &&
+ generatedLabels.some((label) => hasCurrentEnrichmentVersion(label.metadata, ragEnrichmentVersion))
+ );
+ });
+ const documentsMissingCurrentEnrichmentVersion =
+ indexedDocuments.length - documentsWithCurrentEnrichmentVersion.length;
+ const documentsMissingSections = indexedDocuments.filter(
+ (document) => (sectionRowsByDocument.get(document.id) ?? []).length === 0,
+ );
+ const documentsMissingMemoryCards = indexedDocuments.filter(
+ (document) => (memoryRowsByDocument.get(document.id) ?? []).length === 0,
+ );
+ const documentsWithCurrentDeepMemoryVersion = indexedDocuments.filter((document) => {
+ const sections = sectionRowsByDocument.get(document.id) ?? [];
+ const memoryCards = memoryRowsByDocument.get(document.id) ?? [];
+ const chunks = chunkRowsByDocument.get(document.id) ?? [];
+ return (
+ hasCurrentMemoryVersion(document.metadata, ragDeepMemoryVersion) &&
+ sections.some((section) => hasCurrentMemoryVersion(section.metadata, ragDeepMemoryVersion)) &&
+ memoryCards.some((card) => hasCurrentMemoryVersion(card.metadata, ragDeepMemoryVersion)) &&
+ chunks.length > 0 &&
+ chunks.every((chunk) => hasCurrentMemoryVersion(chunk.metadata, ragDeepMemoryVersion))
+ );
+ });
+ const documentsMissingCurrentDeepMemoryVersion =
+ indexedDocuments.length - documentsWithCurrentDeepMemoryVersion.length;
+ const requireCurrentEnrichmentVersion = strictEnrichmentVersionRequired();
+
+ const [
+ missingEmbeddingResult,
+ failedJobsResult,
+ activeJobsResult,
+ imageCountResult,
+ searchableImageResult,
+ oversizedBatchesResult,
+ cleanupIssuesResult,
+ ] = await Promise.all([
+ supabase.from("document_chunks").select("id", { count: "exact", head: true }).is("embedding", null),
+ supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"),
+ supabase
+ .from("ingestion_jobs")
+ .select("id", { count: "exact", head: true })
+ .in("status", ["pending", "processing"]),
+ supabase.from("document_images").select("id", { count: "exact", head: true }),
+ supabase.from("document_images").select("id", { count: "exact", head: true }).eq("searchable", true),
+ supabase
+ .from("import_batches")
+ .select("id,name,total_files,status")
+ .gt("total_files", 150)
+ .in("status", ["queued", "processing"]),
+ supabase
+ .from("storage_cleanup_jobs")
+ .select("id,status,last_error")
+ .in("status", ["pending", "failed"])
+ .order("created_at", { ascending: true })
+ .limit(5),
+ ]);
+
+ for (const result of [
+ missingEmbeddingResult,
+ failedJobsResult,
+ activeJobsResult,
+ imageCountResult,
+ searchableImageResult,
+ oversizedBatchesResult,
+ cleanupIssuesResult,
+ ]) {
+ if (result.error) throw new Error(result.error.message);
+ }
+
+ const issues: string[] = [];
+ if (duplicateGroups.length > 0) issues.push(`duplicate content-hash groups: ${duplicateGroups.length}`);
+ if ((missingEmbeddingResult.count ?? 0) > 0)
+ issues.push(`chunks missing embeddings: ${missingEmbeddingResult.count}`);
+ if (emptyIndexedDocuments.length > 0) issues.push(`empty indexed documents: ${emptyIndexedDocuments.length}`);
+ if (documentsWithChunkCountMismatch.length > 0)
+ issues.push(`indexed document chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`);
+ if (documentsMissingSummaries.length > 0)
+ issues.push(`indexed documents missing summaries: ${documentsMissingSummaries.length}`);
+ if (documentsMissingGeneratedLabels.length > 0)
+ issues.push(`indexed documents missing generated labels: ${documentsMissingGeneratedLabels.length}`);
+ if (documentsMissingSections.length > 0) issues.push(`indexed documents missing sections: ${documentsMissingSections.length}`);
+ if (documentsMissingMemoryCards.length > 0)
+ issues.push(`indexed documents missing memory cards: ${documentsMissingMemoryCards.length}`);
+ if (requireCurrentEnrichmentVersion && documentsMissingCurrentEnrichmentVersion > 0) {
+ issues.push(`indexed documents missing current enrichment version: ${documentsMissingCurrentEnrichmentVersion}`);
+ }
+ if (requireCurrentEnrichmentVersion && documentsMissingCurrentDeepMemoryVersion > 0) {
+ issues.push(`indexed documents missing current deep-memory version: ${documentsMissingCurrentDeepMemoryVersion}`);
+ }
+ if ((failedJobsResult.count ?? 0) > 0) issues.push(`failed ingestion jobs: ${failedJobsResult.count}`);
+ if ((oversizedBatchesResult.data ?? []).length > 0) {
+ issues.push(`active oversized import batches: ${(oversizedBatchesResult.data ?? []).length}`);
+ }
+ if ((cleanupIssuesResult.data ?? []).length > 0) {
+ issues.push(`pending or failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`);
+ }
console.log("Indexing prerequisites ready.");
console.log("Supabase bulk ingestion tables are reachable.");
console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`);
console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`);
+ console.log(`Documents: ${(documents ?? []).length}; empty indexed: ${emptyIndexedDocuments.length}`);
+ console.log(`Chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`);
+ console.log(
+ `Enrichment coverage: summaries missing=${documentsMissingSummaries.length}; generated labels missing=${documentsMissingGeneratedLabels.length}`,
+ );
+ console.log(
+ `RAG enrichment version: ${documentsWithCurrentEnrichmentVersion.length}/${indexedDocuments.length} indexed current (${ragEnrichmentVersion}); strict=${requireCurrentEnrichmentVersion ? "yes" : "no"}`,
+ );
+ console.log(
+ `Deep memory version: ${documentsWithCurrentDeepMemoryVersion.length}/${indexedDocuments.length} indexed current (${ragDeepMemoryVersion}); sections missing=${documentsMissingSections.length}; memory cards missing=${documentsMissingMemoryCards.length}`,
+ );
+ console.log(`Duplicate content-hash groups: ${duplicateGroups.length}`);
+ console.log(`Chunks missing embeddings: ${missingEmbeddingResult.count ?? 0}`);
+ console.log(`Failed jobs: ${failedJobsResult.count ?? 0}; pending/processing jobs: ${activeJobsResult.count ?? 0}`);
+ console.log(`Images: ${imageCountResult.count ?? 0}; searchable: ${searchableImageResult.count ?? 0}`);
+ console.log(`Active oversized batches: ${(oversizedBatchesResult.data ?? []).length}`);
+ console.log(`Pending/failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`);
+
+ if (issues.length > 0) {
+ throw new Error(`Indexing readiness check failed: ${issues.join("; ")}`);
+ }
}
main().catch((error) => {
diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts
new file mode 100644
index 000000000..27389bc13
--- /dev/null
+++ b/scripts/cleanup-storage.ts
@@ -0,0 +1,120 @@
+import { loadEnvConfig } from "@next/env";
+import { loadAdminClient } from "./eval-utils";
+
+loadEnvConfig(process.cwd());
+
+type CleanupArgs = {
+ limit: number;
+ dryRun: boolean;
+};
+
+type CleanupJob = {
+ id: string;
+ document_bucket: string | null;
+ document_paths: string[] | null;
+ image_bucket: string | null;
+ image_paths: string[] | null;
+ attempts: number;
+};
+
+function parseArgs(argv: string[]): CleanupArgs {
+ const args: CleanupArgs = { limit: 50, dryRun: false };
+
+ for (let index = 0; index < argv.length; index += 1) {
+ const token = argv[index];
+ if (token === "--dry-run") {
+ args.dryRun = true;
+ continue;
+ }
+ if (token === "--limit") {
+ const value = argv[index + 1];
+ if (!value || value.startsWith("--")) throw new Error("Missing value for --limit");
+ args.limit = Number.parseInt(value, 10);
+ index += 1;
+ }
+ }
+
+ if (!Number.isInteger(args.limit) || args.limit <= 0) throw new Error("--limit must be a positive integer.");
+ return args;
+}
+
+async function removePaths(args: {
+ supabase: Awaited>;
+ bucket: string;
+ paths: string[];
+}) {
+ let removed = 0;
+ const warnings: string[] = [];
+ const uniquePaths = Array.from(new Set(args.paths.filter(Boolean)));
+
+ for (let start = 0; start < uniquePaths.length; start += 1000) {
+ const batch = uniquePaths.slice(start, start + 1000);
+ const { data, error } = await args.supabase.storage.from(args.bucket).remove(batch);
+ if (error) {
+ warnings.push(`${args.bucket}: ${error.message}`);
+ } else {
+ removed += data?.length ?? 0;
+ }
+ }
+
+ return { removed, warnings };
+}
+
+async function main() {
+ const args = parseArgs(process.argv.slice(2));
+ const supabase = await loadAdminClient();
+ const { data, error } = await supabase
+ .from("storage_cleanup_jobs")
+ .select("id,document_bucket,document_paths,image_bucket,image_paths,attempts")
+ .in("status", ["pending", "failed"])
+ .order("created_at", { ascending: true })
+ .limit(args.limit);
+
+ if (error) throw new Error(error.message);
+ const jobs = (data ?? []) as CleanupJob[];
+ console.log(`Found ${jobs.length} storage cleanup job(s).`);
+ if (args.dryRun || jobs.length === 0) return;
+
+ let completed = 0;
+ let failed = 0;
+
+ for (const job of jobs) {
+ const documentCleanup = await removePaths({
+ supabase,
+ bucket: job.document_bucket ?? "clinical-documents",
+ paths: job.document_paths ?? [],
+ });
+ const imageCleanup = await removePaths({
+ supabase,
+ bucket: job.image_bucket ?? "clinical-images",
+ paths: job.image_paths ?? [],
+ });
+ const warnings = [...documentCleanup.warnings, ...imageCleanup.warnings];
+ const nextStatus = warnings.length > 0 ? "failed" : "completed";
+ const { error: updateError } = await supabase
+ .from("storage_cleanup_jobs")
+ .update({
+ status: nextStatus,
+ attempts: job.attempts + 1,
+ storage_removed: documentCleanup.removed + imageCleanup.removed,
+ last_error: warnings.length ? warnings.join("; ") : null,
+ completed_at: nextStatus === "completed" ? new Date().toISOString() : null,
+ metadata: {
+ operation: "storage_cleanup_retry",
+ storage_warnings: warnings,
+ },
+ })
+ .eq("id", job.id);
+
+ if (updateError) throw new Error(updateError.message);
+ if (nextStatus === "completed") completed += 1;
+ else failed += 1;
+ }
+
+ console.log(`Storage cleanup complete: ${completed} completed, ${failed} failed.`);
+}
+
+main().catch((error) => {
+ console.error(error instanceof Error ? error.message : error);
+ process.exit(1);
+});
diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts
index 35cbd065e..936e59ca8 100644
--- a/scripts/enrich-documents.ts
+++ b/scripts/enrich-documents.ts
@@ -5,12 +5,16 @@ loadEnvConfig(process.cwd());
type EnrichArgs = {
ownerEmail?: string;
+ ownerId?: string;
+ allOwners: boolean;
mode: string;
limit: number;
documentId?: string;
+ includeCurrent: boolean;
};
type SupabaseAdmin = Awaited>;
+type MetadataRow = { id?: string; document_id: string; metadata?: unknown; source?: string | null };
async function loadAdminClient() {
const { createAdminClient } = await import("@/lib/supabase/admin");
@@ -20,18 +24,30 @@ async function loadAdminClient() {
function parseArgs(argv: string[]): EnrichArgs {
const args: EnrichArgs = {
ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL,
+ ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID,
+ allOwners: !process.env.RAG_EVAL_OWNER_EMAIL && !process.env.RAG_EVAL_OWNER_ID && !process.env.LOCAL_NO_AUTH_OWNER_ID,
mode: "summaries-labels-images",
limit: 25,
+ includeCurrent: false,
};
for (let index = 0; index < argv.length; index += 1) {
const token = argv[index];
if (!token.startsWith("--")) continue;
+ if (token === "--all-owners") {
+ args.allOwners = true;
+ continue;
+ }
+ if (token === "--include-current") {
+ args.includeCurrent = true;
+ continue;
+ }
const value = argv[index + 1];
if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
index += 1;
if (token === "--owner-email") args.ownerEmail = value;
+ if (token === "--owner-id") args.ownerId = value;
if (token === "--mode") args.mode = value;
if (token === "--limit") args.limit = Number.parseInt(value, 10);
if (token === "--document-id") args.documentId = value;
@@ -56,15 +72,15 @@ async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) {
throw new Error(`No Supabase Auth user found for ${email}. Sign in once before enriching documents.`);
}
-async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId: string) {
+async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId?: string) {
let query = supabase
.from("documents")
.select("id,owner_id,title,file_name,source_path,status,metadata")
- .eq("owner_id", ownerId)
.eq("status", "indexed")
.order("created_at", { ascending: true })
- .limit(args.limit);
+ .limit(args.documentId ? 1 : Math.max(args.limit * 10, 1000));
+ if (ownerId) query = query.eq("owner_id", ownerId);
if (args.documentId) query = query.eq("id", args.documentId);
const { data, error } = await query;
@@ -72,39 +88,174 @@ async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId:
return data ?? [];
}
+async function loadEnrichmentCoverage(supabase: SupabaseAdmin, documentIds: string[]) {
+ const summaries: MetadataRow[] = [];
+ const labels: MetadataRow[] = [];
+
+ for (let start = 0; start < documentIds.length; start += 100) {
+ const ids = documentIds.slice(start, start + 100);
+ const [summaryResult, labelResult] = await Promise.all([
+ supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids),
+ supabase.from("document_labels").select("id,document_id,source,metadata").in("document_id", ids),
+ ]);
+
+ if (summaryResult.error) throw new Error(summaryResult.error.message);
+ if (labelResult.error) throw new Error(labelResult.error.message);
+ summaries.push(...((summaryResult.data ?? []) as MetadataRow[]));
+ labels.push(...((labelResult.data ?? []) as MetadataRow[]));
+ }
+
+ const coverage = new Map();
+ for (const documentId of documentIds) coverage.set(documentId, { labels: [] });
+ for (const summary of summaries) {
+ coverage.set(summary.document_id, { ...(coverage.get(summary.document_id) ?? { labels: [] }), summary });
+ }
+ for (const label of labels) {
+ const existing = coverage.get(label.document_id) ?? { labels: [] };
+ coverage.set(label.document_id, { ...existing, labels: [...existing.labels, label] });
+ }
+
+ return coverage;
+}
+
+async function loadRowsForDocuments(supabase: SupabaseAdmin, table: string, select: string, documentIds: string[]) {
+ const rows: MetadataRow[] = [];
+ for (let start = 0; start < documentIds.length; start += 100) {
+ const ids = documentIds.slice(start, start + 100);
+ for (let rangeStart = 0; ; rangeStart += 1000) {
+ const { data, error } = await supabase
+ .from(table)
+ .select(select)
+ .in("document_id", ids)
+ .range(rangeStart, rangeStart + 999);
+ if (error) throw new Error(error.message);
+ rows.push(...((data ?? []) as unknown as MetadataRow[]));
+ if (!data || data.length < 1000) break;
+ }
+ }
+ return rows;
+}
+
+async function loadDeepMemoryCoverage(supabase: SupabaseAdmin, documentIds: string[]) {
+ const [sections, memoryCards] = await Promise.all([
+ loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds),
+ loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds),
+ ]);
+
+ const coverage = new Map();
+ for (const documentId of documentIds) coverage.set(documentId, { sections: [], memoryCards: [] });
+ for (const section of sections) {
+ const existing = coverage.get(section.document_id) ?? { sections: [], memoryCards: [] };
+ coverage.set(section.document_id, { ...existing, sections: [...existing.sections, section] });
+ }
+ for (const card of memoryCards) {
+ const existing = coverage.get(card.document_id) ?? { sections: [], memoryCards: [] };
+ coverage.set(card.document_id, { ...existing, memoryCards: [...existing.memoryCards, card] });
+ }
+ return coverage;
+}
+
async function loadEvidence(supabase: SupabaseAdmin, documentId: string) {
- const [chunksResult, imagesResult] = await Promise.all([
- supabase
+ const chunks = [];
+ const images = [];
+
+ for (let start = 0; ; start += 1000) {
+ const { data, error } = await supabase
.from("document_chunks")
- .select("id,page_number,chunk_index,section_heading,content")
+ .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata")
.eq("document_id", documentId)
.order("chunk_index", { ascending: true })
- .limit(24),
- supabase
+ .range(start, start + 999);
+ if (error) throw new Error(error.message);
+ chunks.push(...(data ?? []));
+ if (!data || data.length < 1000) break;
+ }
+
+ for (let start = 0; ; start += 1000) {
+ const { data, error } = await supabase
.from("document_images")
- .select("id,page_number,caption,image_type,labels")
+ .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata")
.eq("document_id", documentId)
.eq("searchable", true)
.order("clinical_relevance_score", { ascending: false })
- .limit(12),
- ]);
+ .range(start, start + 999);
+ if (error) throw new Error(error.message);
+ images.push(...(data ?? []));
+ if (!data || data.length < 1000) break;
+ }
- if (chunksResult.error) throw new Error(chunksResult.error.message);
- if (imagesResult.error) throw new Error(imagesResult.error.message);
- return { chunks: chunksResult.data ?? [], images: imagesResult.data ?? [] };
+ return { chunks, images };
}
function hashBytes(bytes: Buffer) {
return createHash("sha256").update(bytes).digest("hex");
}
+function metadataString(metadata: unknown, key: string) {
+ if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return null;
+ const value = (metadata as Record)[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function metadataRecord(metadata: unknown): Record {
+ return metadata && typeof metadata === "object" && !Array.isArray(metadata)
+ ? { ...(metadata as Record) }
+ : {};
+}
+
+function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) {
+ return metadataRecord(metadata).rag_enrichment_version === expectedVersion;
+}
+
+function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) {
+ const record = metadataRecord(metadata);
+ return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion;
+}
+
+function needsEnrichmentBackfill(args: {
+ document: { metadata?: unknown };
+ coverage?: { summary?: MetadataRow; labels: MetadataRow[] };
+ memoryCoverage?: { sections: MetadataRow[]; memoryCards: MetadataRow[] };
+ ragEnrichmentVersion: string;
+ ragDeepMemoryVersion: string;
+}) {
+ const generatedLabels = args.coverage?.labels.filter((label) => label.source === "generated") ?? [];
+ const sections = args.memoryCoverage?.sections ?? [];
+ const memoryCards = args.memoryCoverage?.memoryCards ?? [];
+ const summaryMetadata = metadataRecord(args.coverage?.summary?.metadata);
+ const documentMetadata = metadataRecord(args.document.metadata);
+ return (
+ !args.coverage?.summary ||
+ generatedLabels.length === 0 ||
+ !summaryMetadata.coverage_profile ||
+ !documentMetadata.coverage_profile ||
+ !hasCurrentEnrichmentVersion(args.document.metadata, args.ragEnrichmentVersion) ||
+ !hasCurrentEnrichmentVersion(args.coverage.summary?.metadata, args.ragEnrichmentVersion) ||
+ generatedLabels.every((label) => !hasCurrentEnrichmentVersion(label.metadata, args.ragEnrichmentVersion)) ||
+ sections.length === 0 ||
+ memoryCards.length === 0 ||
+ !hasCurrentMemoryVersion(args.document.metadata, args.ragDeepMemoryVersion) ||
+ sections.every((section) => !hasCurrentMemoryVersion(section.metadata, args.ragDeepMemoryVersion)) ||
+ memoryCards.every((card) => !hasCurrentMemoryVersion(card.metadata, args.ragDeepMemoryVersion))
+ );
+}
+
async function classifyExistingImages(supabase: SupabaseAdmin, documentId: string) {
- const [{ env }, { cheapImageSkipReason, classifiedImageSkipReason, lightweightPerceptualHash }, { classifyAndCaptionImageFromBase64 }] =
- await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]);
+ const [
+ { env },
+ {
+ assessClinicalImageUse,
+ cheapImageSkipReason,
+ classifiedImageSkipReason,
+ clinicalImagePolicyVersion,
+ lightweightPerceptualHash,
+ },
+ { classifyAndCaptionImageFromBase64 },
+ ] = await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]);
const { data: images, error } = await supabase
.from("document_images")
.select(
- "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason",
+ "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason,labels,metadata",
)
.eq("document_id", documentId)
.order("page_number", { ascending: true });
@@ -116,11 +267,8 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
let skipped = 0;
for (const image of images ?? []) {
- if (
- (image.image_type && image.image_type !== "unclear") ||
- image.clinical_relevance_score > 0 ||
- image.skip_reason
- ) {
+ const existingMetadata = metadataRecord(image.metadata);
+ if (existingMetadata.image_policy_version === clinicalImagePolicyVersion) {
if (image.searchable) searchable += 1;
else skipped += 1;
continue;
@@ -147,7 +295,13 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
bbox: image.bbox as [number, number, number, number] | null,
width: image.width,
height: image.height,
- sourceKind: image.source_kind as "embedded" | "diagram_crop" | "page_region" | "fallback" | undefined,
+ sourceKind: image.source_kind as
+ | "embedded"
+ | "table_crop"
+ | "diagram_crop"
+ | "page_region"
+ | "fallback"
+ | undefined,
},
});
@@ -167,70 +321,238 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
}
seenHashes.add(imageHash);
- const classification = await classifyAndCaptionImageFromBase64({
- base64: bytes.toString("base64"),
- mimeType: image.mime_type,
- nearbyText: image.caption ?? undefined,
+ const baseAssessment = assessClinicalImageUse({
+ imageType: image.image_type,
+ searchable: image.searchable,
+ clinicalRelevanceScore: image.clinical_relevance_score,
+ sourceKind: image.source_kind,
+ tableRole: metadataString(image.metadata, "table_role"),
+ tableText: metadataString(image.metadata, "table_text"),
+ tableTitle: metadataString(image.metadata, "table_title"),
+ tableLabel: metadataString(image.metadata, "table_label"),
+ caption: image.caption,
+ labels: Array.isArray(image.labels) ? image.labels : [],
+ skipReason: image.skip_reason,
+ });
+ const classification =
+ image.source_kind === "table_crop" && ["administrative", "reference"].includes(baseAssessment.clinical_use_class)
+ ? {
+ image_type: image.image_type || "clinical_table",
+ searchable: false,
+ clinical_relevance_score: 0,
+ labels: [],
+ caption:
+ baseAssessment.clinical_use_class === "administrative"
+ ? "Administrative document-control table retained for audit, not clinical evidence."
+ : "Reference table retained for audit, not clinical evidence.",
+ skip_reason: baseAssessment.clinical_use_reason,
+ clinical_use_class: baseAssessment.clinical_use_class,
+ clinical_use_reason: baseAssessment.clinical_use_reason,
+ clinical_signal_score: baseAssessment.clinical_signal_score,
+ admin_signal_score: baseAssessment.admin_signal_score,
+ }
+ : await classifyAndCaptionImageFromBase64({
+ base64: bytes.toString("base64"),
+ mimeType: image.mime_type,
+ nearbyText: image.caption ?? undefined,
+ sourceKind: image.source_kind,
+ candidateType: metadataString(image.metadata, "candidate_type"),
+ tableLabel: metadataString(image.metadata, "table_label"),
+ tableTitle: metadataString(image.metadata, "table_title"),
+ tableRole: metadataString(image.metadata, "table_role"),
+ tableText: metadataString(image.metadata, "table_text"),
+ });
+ const finalAssessment = assessClinicalImageUse({
+ imageType: classification.image_type,
+ searchable: classification.searchable,
+ clinicalRelevanceScore: classification.clinical_relevance_score,
+ sourceKind: image.source_kind,
+ tableRole: metadataString(image.metadata, "table_role"),
+ tableText: metadataString(image.metadata, "table_text"),
+ tableTitle: metadataString(image.metadata, "table_title"),
+ tableLabel: metadataString(image.metadata, "table_label"),
+ caption: classification.caption,
+ labels: classification.labels,
+ skipReason: classification.skip_reason,
});
- const classifiedSkip = classifiedImageSkipReason(classification);
+ const classifiedSkip = classifiedImageSkipReason({
+ ...classification,
+ searchable: finalAssessment.searchable,
+ clinical_relevance_score: finalAssessment.clinical_relevance_score,
+ clinical_use_class: finalAssessment.clinical_use_class,
+ clinical_use_reason: finalAssessment.clinical_use_reason,
+ clinical_signal_score: finalAssessment.clinical_signal_score,
+ admin_signal_score: finalAssessment.admin_signal_score,
+ });
+ const retainAsAuditTable =
+ image.source_kind === "table_crop" &&
+ ["administrative", "reference"].includes(finalAssessment.clinical_use_class) &&
+ classification.image_type !== "logo_decorative";
+ const nextSearchable = finalAssessment.searchable;
await supabase
.from("document_images")
.update({
caption: classification.caption || image.caption,
image_type: classification.image_type,
- searchable: !classifiedSkip,
- clinical_relevance_score: classifiedSkip ? 0 : classification.clinical_relevance_score,
- skip_reason: classifiedSkip,
+ searchable: nextSearchable,
+ clinical_relevance_score: nextSearchable ? finalAssessment.clinical_relevance_score : 0,
+ skip_reason: nextSearchable ? null : classifiedSkip,
image_hash: imageHash,
perceptual_hash: perceptualHash,
labels: classification.labels,
+ metadata: {
+ ...metadataRecord(image.metadata),
+ clinical_use_class: finalAssessment.clinical_use_class,
+ clinical_use_reason: finalAssessment.clinical_use_reason,
+ clinical_signal_score: finalAssessment.clinical_signal_score,
+ admin_signal_score: finalAssessment.admin_signal_score,
+ image_policy_version: clinicalImagePolicyVersion,
+ retained_for_audit: retainAsAuditTable || undefined,
+ retained_for_document_view: retainAsAuditTable || undefined,
+ },
})
.eq("id", image.id);
- if (classifiedSkip) skipped += 1;
+ if (!nextSearchable) skipped += 1;
else searchable += 1;
}
return { searchable, skipped, total: images?.length ?? 0 };
}
+async function stampExistingEnrichmentVersion(args: {
+ supabase: SupabaseAdmin;
+ document: { id: string; metadata?: unknown };
+ coverage: { summary?: MetadataRow; labels: MetadataRow[] };
+ ragEnrichmentVersion: string;
+}) {
+ const stampedAt = new Date().toISOString();
+ const marker = {
+ rag_enrichment_version: args.ragEnrichmentVersion,
+ rag_enrichment_updated_at: stampedAt,
+ version_stamped_at: stampedAt,
+ };
+
+ const { error: documentError } = await args.supabase
+ .from("documents")
+ .update({ metadata: { ...metadataRecord(args.document.metadata), ...marker } })
+ .eq("id", args.document.id);
+ if (documentError) throw new Error(documentError.message);
+
+ if (args.coverage.summary) {
+ const { error: summaryError } = await args.supabase
+ .from("document_summaries")
+ .update({ metadata: { ...metadataRecord(args.coverage.summary.metadata), ...marker } })
+ .eq("document_id", args.document.id);
+ if (summaryError) throw new Error(summaryError.message);
+ }
+
+ for (const label of args.coverage.labels.filter((item) => item.source === "generated")) {
+ if (!label.id) continue;
+ const { error: labelError } = await args.supabase
+ .from("document_labels")
+ .update({ metadata: { ...metadataRecord(label.metadata), ...marker } })
+ .eq("id", label.id);
+ if (labelError) throw new Error(labelError.message);
+ }
+}
+
async function main() {
const args = parseArgs(process.argv.slice(2));
- if (!args.ownerEmail) {
- throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.');
+ if (!args.ownerId && !args.ownerEmail && !args.allOwners) {
+ throw new Error(
+ 'Provide --owner-id, set LOCAL_NO_AUTH_OWNER_ID or RAG_EVAL_OWNER_ID, provide --owner-email "you@example.com", or pass --all-owners.',
+ );
}
- if (args.mode !== "summaries-labels-images") {
- throw new Error("--mode currently supports summaries-labels-images.");
+ if (!["summaries-labels-images", "metadata-stamp", "deep-memory"].includes(args.mode)) {
+ throw new Error("--mode supports summaries-labels-images, deep-memory, or metadata-stamp.");
}
- const [{ requireOpenAIEnv, requireServerEnv }, { upsertDocumentEnrichment }, supabase] = await Promise.all([
+ const [
+ { requireOpenAIEnv, requireServerEnv },
+ { ragEnrichmentVersion, upsertDocumentEnrichment },
+ { ragDeepMemoryVersion, upsertDocumentDeepMemory },
+ supabase,
+ ] = await Promise.all([
import("@/lib/env"),
import("@/lib/document-enrichment"),
+ import("@/lib/deep-memory"),
loadAdminClient(),
]);
requireServerEnv();
- requireOpenAIEnv();
+ if (args.mode === "summaries-labels-images" || args.mode === "deep-memory") requireOpenAIEnv();
+
+ const ownerId =
+ !args.allOwners && args.ownerId
+ ? args.ownerId
+ : args.ownerEmail && !args.allOwners
+ ? await findOwnerIdByEmail(supabase, args.ownerEmail)
+ : undefined;
+ const loadedDocuments = await loadDocuments(supabase, args, ownerId);
+ const coverage = await loadEnrichmentCoverage(
+ supabase,
+ loadedDocuments.map((document) => document.id),
+ );
+ const memoryCoverage = await loadDeepMemoryCoverage(
+ supabase,
+ loadedDocuments.map((document) => document.id),
+ );
+ const documents = loadedDocuments
+ .filter((document) =>
+ args.includeCurrent
+ ? true
+ : needsEnrichmentBackfill({
+ document,
+ coverage: coverage.get(document.id),
+ memoryCoverage: memoryCoverage.get(document.id),
+ ragEnrichmentVersion,
+ ragDeepMemoryVersion,
+ }),
+ )
+ .slice(0, args.limit);
- const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail);
- const documents = await loadDocuments(supabase, args, ownerId);
- console.log(`Enriching ${documents.length} indexed document(s).`);
+ console.log(
+ `Enriching ${documents.length} indexed document(s). scope=${ownerId ? "owner" : "all"} version=${ragEnrichmentVersion}`,
+ );
let completed = 0;
for (const document of documents) {
- const imageStats = await classifyExistingImages(supabase, document.id);
- await supabase
- .from("documents")
- .update({
- image_count: imageStats.searchable,
- metadata: {
- ...(document.metadata ?? {}),
- searchable_image_count: imageStats.searchable,
- skipped_image_count: imageStats.skipped,
- image_enriched_at: new Date().toISOString(),
- },
- })
- .eq("id", document.id);
+ const documentCoverage = coverage.get(document.id);
+ if (args.mode === "metadata-stamp") {
+ if (!documentCoverage?.summary || documentCoverage.labels.every((label) => label.source !== "generated")) {
+ console.log(`SKIP cannot metadata-stamp missing enrichment: ${document.file_name}`);
+ continue;
+ }
+ await stampExistingEnrichmentVersion({
+ supabase,
+ document,
+ coverage: documentCoverage,
+ ragEnrichmentVersion,
+ });
+ completed += 1;
+ console.log(`STAMPED ${document.file_name}`);
+ continue;
+ }
+
+ let imageMetadata = metadataRecord(document.metadata);
+ let imageStats = { searchable: 0, skipped: 0, total: 0 };
+ if (args.mode === "summaries-labels-images") {
+ imageStats = await classifyExistingImages(supabase, document.id);
+ imageMetadata = {
+ ...imageMetadata,
+ searchable_image_count: imageStats.searchable,
+ skipped_image_count: imageStats.skipped,
+ image_enriched_at: new Date().toISOString(),
+ };
+ await supabase
+ .from("documents")
+ .update({
+ image_count: imageStats.searchable,
+ metadata: imageMetadata,
+ })
+ .eq("id", document.id);
+ }
const evidence = await loadEvidence(supabase, document.id);
if (evidence.chunks.length === 0) {
@@ -238,15 +560,23 @@ async function main() {
continue;
}
- await upsertDocumentEnrichment({
+ if (args.mode === "summaries-labels-images") {
+ await upsertDocumentEnrichment({
+ supabase,
+ document: { ...document, metadata: imageMetadata },
+ chunks: evidence.chunks,
+ images: evidence.images,
+ });
+ }
+ const deepMemory = await upsertDocumentDeepMemory({
supabase,
- document,
+ document: { ...document, metadata: imageMetadata },
chunks: evidence.chunks,
images: evidence.images,
});
completed += 1;
console.log(
- `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`,
+ `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} sections=${deepMemory.sections.length} memory=${deepMemory.memoryCards.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`,
);
}
diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts
index 93cdb1836..d050c0a91 100644
--- a/scripts/eval-rag.ts
+++ b/scripts/eval-rag.ts
@@ -1,18 +1,13 @@
import { loadEnvConfig } from "@next/env";
import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases";
import type { RagAnswer } from "@/lib/types";
-import {
- estimateCostUsd,
- findOwnerIdByEmail,
- loadAdminClient,
- percentile,
- validateRagAnswer,
-} from "./eval-utils";
+import { estimateCostUsd, findOwnerIdByEmail, loadAdminClient, percentile, validateRagAnswer } from "./eval-utils";
loadEnvConfig(process.cwd());
type EvalArgs = {
ownerEmail?: string;
+ ownerId?: string;
limit?: number;
question?: string;
json: boolean;
@@ -44,6 +39,7 @@ type EvalResult = {
function parseArgs(argv: string[]): EvalArgs {
const args: EvalArgs = {
ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL,
+ ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID,
json: false,
failOnThreshold: false,
};
@@ -66,6 +62,7 @@ function parseArgs(argv: string[]): EvalArgs {
index += 1;
if (token === "--owner-email") args.ownerEmail = value;
+ if (token === "--owner-id") args.ownerId = value;
if (token === "--limit") args.limit = Number.parseInt(value, 10);
if (token === "--question") args.question = value;
}
@@ -90,12 +87,13 @@ function summarizeFailures(results: EvalResult[]) {
const failedCases = results.filter((result) => result.failures.length > 0);
const failures: string[] = [];
- if (supported.length >= 18 && groundedSupported < 17) failures.push(`supported grounded ${groundedSupported}/${supported.length}`);
+ if (supported.length >= 18 && groundedSupported < 17)
+ failures.push(`supported grounded ${groundedSupported}/${supported.length}`);
if (unsupported.length >= 2 && unsupportedCorrect !== unsupported.length) {
failures.push(`unsupported correct ${unsupportedCorrect}/${unsupported.length}`);
}
if (invalidCitations > 0) failures.push(`invalid citation count ${invalidCitations}`);
- if (routineExtractiveLatencies.length > 0 && percentile(routineExtractiveLatencies, 95) > 2000) {
+ if (routineExtractiveLatencies.length >= 5 && percentile(routineExtractiveLatencies, 95) > 2000) {
failures.push("routine extractive p95 over 2000ms");
}
if (complexSlow > 0) failures.push(`complex answers over 20000ms: ${complexSlow}`);
@@ -147,14 +145,15 @@ function printHumanSummary(results: EvalResult[]) {
console.log(` token_totals=${JSON.stringify(tokenTotals)}`);
console.log(
` estimated_cost_usd=${
- estimatedCostUsd === null ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION" : estimatedCostUsd.toFixed(6)
+ estimatedCostUsd === null
+ ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION"
+ : estimatedCostUsd.toFixed(6)
}`,
);
}
async function main() {
const args = parseArgs(process.argv.slice(2));
- if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.');
const [{ requireOpenAIEnv, requireServerEnv }, { answerQuestionWithScope }, supabase] = await Promise.all([
import("@/lib/env"),
@@ -165,11 +164,12 @@ async function main() {
requireServerEnv();
requireOpenAIEnv();
- const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail);
+ const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
+ const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public";
const cases = selectRagEvalCases({ limit: args.limit, question: args.question });
const results: EvalResult[] = [];
- if (!args.json) console.log(`Running ${cases.length} RAG eval case(s).`);
+ if (!args.json) console.log(`Running ${cases.length} RAG eval case(s), scope=${scope}.`);
for (const testCase of cases) {
const answer = (await answerQuestionWithScope({
@@ -225,7 +225,7 @@ async function main() {
const thresholdFailures = summarizeFailures(results);
if (args.json) {
- console.log(JSON.stringify({ results, thresholdFailures }, null, 2));
+ console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2));
} else {
printHumanSummary(results);
if (thresholdFailures.length > 0) {
diff --git a/scripts/eval-search.ts b/scripts/eval-search.ts
index 7c16c6151..0448bd054 100644
--- a/scripts/eval-search.ts
+++ b/scripts/eval-search.ts
@@ -1,25 +1,37 @@
import { loadEnvConfig } from "@next/env";
+import { pathToFileURL } from "node:url";
import { buildVisualEvidence } from "@/lib/evidence";
import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases";
import type { SearchResult } from "@/lib/types";
-import { expectedFileHit, findOwnerIdByEmail, hasInvalidVisualEvidence, loadAdminClient, percentile } from "./eval-utils";
+import {
+ expectedFileCoverage,
+ expectedFileHit,
+ findOwnerIdByEmail,
+ hasInvalidVisualEvidence,
+ loadAdminClient,
+ percentile,
+} from "./eval-utils";
loadEnvConfig(process.cwd());
type EvalArgs = {
ownerEmail?: string;
+ ownerId?: string;
limit?: number;
question?: string;
json: boolean;
failOnThreshold: boolean;
};
-type SearchEvalResult = {
+export type SearchEvalResult = {
id: string;
question: string;
category: RagEvalCase["category"];
supported: boolean;
+ expectedFileCount: number;
expectedHitTop3: boolean;
+ expectedAllHitTop5: boolean | null;
+ missingExpectedFiles: string[];
resultCount: number;
topScore: number;
topFiles: string[];
@@ -36,6 +48,7 @@ type SearchEvalResult = {
function parseArgs(argv: string[]): EvalArgs {
const args: EvalArgs = {
ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL,
+ ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID,
json: false,
failOnThreshold: false,
};
@@ -58,6 +71,7 @@ function parseArgs(argv: string[]): EvalArgs {
index += 1;
if (token === "--owner-email") args.ownerEmail = value;
+ if (token === "--owner-id") args.ownerId = value;
if (token === "--limit") args.limit = Number.parseInt(value, 10);
if (token === "--question") args.question = value;
}
@@ -76,26 +90,34 @@ function topScore(results: SearchResult[]) {
function validateSearchResult(testCase: RagEvalCase, result: Omit) {
const failures: string[] = [];
- if (testCase.supported && !result.expectedHitTop3) failures.push("expected document not in top 3");
+ if (testCase.supported && result.expectedFileCount > 1 && result.expectedAllHitTop5 === false) {
+ failures.push(`expected documents missing from top 5: ${result.missingExpectedFiles.join(", ")}`);
+ } else if (testCase.supported && result.expectedFileCount === 1 && !result.expectedHitTop3) {
+ failures.push("expected document not in top 3");
+ }
if (testCase.supported && result.resultCount === 0) failures.push("expected search results");
- if (!testCase.supported && result.topScore >= 0.82) failures.push("unsupported control produced high-confidence match");
- if (testCase.category === "routine" && result.latencyMs > 1500) failures.push("routine search over 1500ms");
-
+ if (!testCase.supported && result.topScore >= 0.82)
+ failures.push("unsupported control produced high-confidence match");
return failures;
}
-function summarizeFailures(results: SearchEvalResult[]) {
+export function summarizeFailures(results: SearchEvalResult[]) {
const routine = results.filter((result) => result.category === "routine");
const supported = results.filter((result) => result.supported);
const unsupported = results.filter((result) => !result.supported);
+ const expectedCoverageCases = supported.filter((result) => result.expectedFileCount > 0);
const routineLatencies = routine.map((result) => result.latencyMs);
- const expectedHits = results.filter((result) => result.expectedHitTop3).length;
+ const expectedHits = expectedCoverageCases.filter((result) =>
+ result.expectedFileCount > 1 ? result.expectedAllHitTop5 : result.expectedHitTop3,
+ ).length;
const routineEmbeddingSkipped = routine.filter((result) => result.embeddingSkipped).length;
const unsupportedHighConfidence = unsupported.filter((result) => result.topScore >= 0.82).length;
const failures: string[] = [];
- if (expectedHits < 18) failures.push(`expected document top-3 hit ${expectedHits}/${results.length}`);
- if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500) failures.push("routine search p95 over 1500ms");
+ if (expectedCoverageCases.length >= 18 && expectedHits < expectedCoverageCases.length)
+ failures.push(`expected document coverage ${expectedHits}/${expectedCoverageCases.length}`);
+ if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500)
+ failures.push("routine search p95 over 1500ms");
if (routine.length > 0 && routineEmbeddingSkipped / routine.length < 0.7) {
failures.push(`routine embedding skipped ${routineEmbeddingSkipped}/${routine.length}`);
}
@@ -108,7 +130,12 @@ function summarizeFailures(results: SearchEvalResult[]) {
function printHumanSummary(results: SearchEvalResult[]) {
const latencies = results.map((result) => result.latencyMs);
const routineLatencies = results.filter((result) => result.category === "routine").map((result) => result.latencyMs);
- const expectedHits = results.filter((result) => result.expectedHitTop3).length;
+ const expectedCoverageCases = results.filter((result) => result.supported && result.expectedFileCount > 0);
+ const expectedHits = expectedCoverageCases.filter((result) =>
+ result.expectedFileCount > 1 ? result.expectedAllHitTop5 : result.expectedHitTop3,
+ ).length;
+ const multiDocumentCoverage = expectedCoverageCases.filter((result) => result.expectedFileCount > 1);
+ const multiDocumentHits = multiDocumentCoverage.filter((result) => result.expectedAllHitTop5).length;
const textFast = results.filter((result) => result.retrievalStrategy === "text_fast_path").length;
const embeddingSkipped = results.filter((result) => result.embeddingSkipped).length;
const fallbackToEmbedding = results.filter((result) => result.fallbackToEmbedding).length;
@@ -120,7 +147,8 @@ function printHumanSummary(results: SearchEvalResult[]) {
console.log("");
console.log("Search eval summary:");
- console.log(` expected_document_hit_top3=${expectedHits}/${results.length}`);
+ console.log(` expected_document_coverage=${expectedHits}/${expectedCoverageCases.length}`);
+ console.log(` multi_document_all_expected_top5=${multiDocumentHits}/${multiDocumentCoverage.length}`);
console.log(` average_latency_ms=${Math.round(latencies.reduce((sum, value) => sum + value, 0) / results.length)}`);
console.log(` p50_latency_ms=${percentile(latencies, 50)}`);
console.log(` p95_latency_ms=${percentile(latencies, 95)}`);
@@ -133,7 +161,6 @@ function printHumanSummary(results: SearchEvalResult[]) {
async function main() {
const args = parseArgs(process.argv.slice(2));
- if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.');
const [{ requireOpenAIEnv, requireServerEnv }, { searchChunksWithTelemetry }, supabase] = await Promise.all([
import("@/lib/env"),
@@ -144,11 +171,12 @@ async function main() {
requireServerEnv();
requireOpenAIEnv();
- const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail);
+ const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
+ const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public";
const cases = selectRagEvalCases({ limit: args.limit, question: args.question });
const results: SearchEvalResult[] = [];
- if (!args.json) console.log(`Running ${cases.length} search eval case(s).`);
+ if (!args.json) console.log(`Running ${cases.length} search eval case(s), scope=${scope}.`);
for (const testCase of cases) {
const startedAt = Date.now();
@@ -160,15 +188,21 @@ async function main() {
skipCache: true,
});
const latencyMs =
- search.telemetry.supabase_rpc_latency_ms + search.telemetry.embedding_latency_ms + search.telemetry.rerank_latency_ms ||
- Date.now() - startedAt;
+ search.telemetry.supabase_rpc_latency_ms +
+ search.telemetry.embedding_latency_ms +
+ search.telemetry.rerank_latency_ms || Date.now() - startedAt;
const visuals = buildVisualEvidence(search.results);
+ const top3Coverage = expectedFileCoverage(testCase.expectedFiles, search.results, 3);
+ const top5Coverage = expectedFileCoverage(testCase.expectedFiles, search.results, 5);
const baseResult = {
id: testCase.id,
question: testCase.question,
category: testCase.category,
supported: testCase.supported,
+ expectedFileCount: testCase.expectedFiles.length,
expectedHitTop3: expectedFileHit(testCase.expectedFiles, search.results, 3),
+ expectedAllHitTop5: testCase.expectedFiles.length > 1 ? top5Coverage.allHit : null,
+ missingExpectedFiles: testCase.expectedFiles.length > 1 ? top5Coverage.missingFiles : top3Coverage.missingFiles,
resultCount: search.results.length,
topScore: topScore(search.results),
topFiles: Array.from(new Set(search.results.slice(0, 3).map((result) => result.file_name))),
@@ -187,8 +221,10 @@ async function main() {
if (!args.json) {
const failureSuffix = result.failures.length ? ` FAIL=${result.failures.join("; ")}` : "";
+ const expectedCoverage =
+ result.expectedFileCount > 1 ? ` allExpectedTop5=${result.expectedAllHitTop5}` : "";
console.log(
- `SEARCH ${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} skippedEmbedding=${result.embeddingSkipped} expectedHit=${result.expectedHitTop3} topScore=${result.topScore.toFixed(3)}${failureSuffix}`,
+ `SEARCH ${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} skippedEmbedding=${result.embeddingSkipped} expectedHit=${result.expectedHitTop3}${expectedCoverage} topScore=${result.topScore.toFixed(3)}${failureSuffix}`,
);
console.log(` Q: ${testCase.question}`);
console.log(` Top files: ${result.topFiles.join("; ") || "none"}`);
@@ -198,7 +234,7 @@ async function main() {
const thresholdFailures = summarizeFailures(results);
if (args.json) {
- console.log(JSON.stringify({ results, thresholdFailures }, null, 2));
+ console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2));
} else {
printHumanSummary(results);
if (thresholdFailures.length > 0) {
@@ -209,7 +245,9 @@ async function main() {
if (args.failOnThreshold && thresholdFailures.length > 0) process.exit(1);
}
-main().catch((error) => {
- console.error(error instanceof Error ? error.message : error);
- process.exit(1);
-});
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ main().catch((error) => {
+ console.error(error instanceof Error ? error.message : error);
+ process.exit(1);
+ });
+}
diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts
index fa28faa25..261ea60a8 100644
--- a/scripts/eval-utils.ts
+++ b/scripts/eval-utils.ts
@@ -30,10 +30,35 @@ export function percentile(values: number[], percentileValue: number) {
return sorted[index];
}
-export function expectedFileHit(expectedFiles: string[], sources: Array>, limit = 3) {
- if (expectedFiles.length === 0) return true;
+export type ExpectedFileCoverage = {
+ expectedFiles: string[];
+ matchedFiles: string[];
+ missingFiles: string[];
+ anyHit: boolean;
+ allHit: boolean;
+};
+
+export function expectedFileCoverage(
+ expectedFiles: string[],
+ sources: Array>,
+ limit = 3,
+): ExpectedFileCoverage {
const topFiles = sources.slice(0, limit).map((source) => source.file_name.toLowerCase());
- return expectedFiles.some((expected) => topFiles.some((file) => file.includes(expected.toLowerCase())));
+ const matchedFiles = expectedFiles.filter((expected) =>
+ topFiles.some((file) => file.includes(expected.toLowerCase())),
+ );
+
+ return {
+ expectedFiles,
+ matchedFiles,
+ missingFiles: expectedFiles.filter((expected) => !matchedFiles.includes(expected)),
+ anyHit: matchedFiles.length > 0,
+ allHit: expectedFiles.length > 0 && matchedFiles.length === expectedFiles.length,
+ };
+}
+
+export function expectedFileHit(expectedFiles: string[], sources: Array>, limit = 3) {
+ return expectedFileCoverage(expectedFiles, sources, limit).anyHit;
}
export function hasInvalidVisualEvidence(cards: VisualEvidenceCard[] = []) {
@@ -42,22 +67,32 @@ export function hasInvalidVisualEvidence(cards: VisualEvidenceCard[] = []) {
export function validateRagAnswer(testCase: RagEvalCase, answer: RagAnswer) {
const failures: string[] = [];
- const expectedHit = expectedFileHit(testCase.expectedFiles, answer.sources);
+ const expectedCoverage = expectedFileCoverage(
+ testCase.expectedFiles,
+ answer.sources,
+ testCase.expectedFiles.length > 1 ? 5 : 3,
+ );
+ const expectedHit = testCase.expectedFiles.length > 1 ? expectedCoverage.allHit : expectedCoverage.anyHit;
const route = answer.routingMode ?? "unsupported";
const visualEvidence = answer.visualEvidence ?? [];
if (testCase.supported && !answer.grounded) failures.push("expected grounded answer");
if (!testCase.supported && answer.grounded) failures.push("expected unsupported answer");
if (!testCase.allowedRoutes.includes(route)) failures.push(`unexpected route ${route}`);
- if (answer.citations.length < testCase.minCitations) failures.push(`expected at least ${testCase.minCitations} citations`);
- if (testCase.expectedFiles.length > 0 && !expectedHit) failures.push("expected document not in retrieved sources");
+ if (answer.citations.length < testCase.minCitations)
+ failures.push(`expected at least ${testCase.minCitations} citations`);
+ if (testCase.expectedFiles.length > 1 && !expectedCoverage.allHit) {
+ failures.push(`expected documents missing from top 5: ${expectedCoverage.missingFiles.join(", ")}`);
+ } else if (testCase.expectedFiles.length === 1 && !expectedCoverage.anyHit) {
+ failures.push("expected document not in retrieved sources");
+ }
if (testCase.requireVisualEvidence && visualEvidence.length === 0) failures.push("expected visual evidence");
if (hasInvalidVisualEvidence(visualEvidence)) failures.push("decorative or zero-relevance visual evidence returned");
if (testCase.category === "complex" && (answer.latencyTimings?.total_latency_ms ?? 0) > testCase.latencyTargetMs) {
failures.push(`latency over ${testCase.latencyTargetMs}ms`);
}
- return { expectedHit, failures };
+ return { expectedHit, expectedCoverage, failures };
}
export function configuredCostRates() {
diff --git a/scripts/import-documents.ts b/scripts/import-documents.ts
index e57569583..f9e3dc47e 100644
--- a/scripts/import-documents.ts
+++ b/scripts/import-documents.ts
@@ -8,8 +8,8 @@ import {
type ExistingImportDocument,
parseImportCliArgs,
scanImportFiles,
- titleFromFileName,
} from "@/lib/bulk-import";
+import { planDocumentName } from "@/lib/document-naming";
loadEnvConfig(process.cwd());
@@ -118,7 +118,15 @@ async function main() {
if (!files.length) return;
if (args.dryRun) {
let existing = new Map();
- if (args.ownerEmail) {
+ const dryRunOwnerId = args.ownerId ?? process.env.LOCAL_NO_AUTH_OWNER_ID;
+ if (dryRunOwnerId) {
+ const supabase = await loadAdminClient();
+ existing = await existingDocumentsByHash(
+ supabase,
+ dryRunOwnerId,
+ Array.from(new Set(files.map((file) => file.contentHash))),
+ );
+ } else if (args.ownerEmail) {
const supabase = await loadAdminClient();
const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail);
existing = await existingDocumentsByHash(
@@ -138,7 +146,7 @@ async function main() {
);
}
if (files.length > 20) console.log(`DRY RUN omitted ${files.length - 20} additional file(s).`);
- if (args.ownerEmail) {
+ if (dryRunOwnerId || args.ownerEmail) {
console.log(
`DRY RUN duplicate check: would_queue=${files.length - exactCopyCount}, exact_copies=${exactCopyCount}, total=${files.length}`,
);
@@ -147,11 +155,12 @@ async function main() {
}
const [{ env }, supabase] = await Promise.all([import("@/lib/env"), loadAdminClient()]);
- if (!args.ownerEmail && !args.resume) {
- throw new Error("Provide --owner-email for a new import batch.");
+ const configuredOwnerId = args.ownerId ?? process.env.LOCAL_NO_AUTH_OWNER_ID;
+ if (!configuredOwnerId && !args.ownerEmail && !args.resume) {
+ throw new Error("Provide --owner-id, set LOCAL_NO_AUTH_OWNER_ID, or provide --owner-email for a new import batch.");
}
- const requestedOwnerId = args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined;
+ const requestedOwnerId = configuredOwnerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
const batch = await loadOrCreateBatch({
supabase,
ownerId: requestedOwnerId,
@@ -216,16 +225,25 @@ async function main() {
const documentId = createDocumentId();
const storagePath = buildImportStoragePath(ownerId, documentId, file.fileName);
- const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, await readFile(file.absolutePath), {
- contentType: "application/pdf",
- upsert: false,
+ const namePlan = await planDocumentName({
+ supabase: supabase as never,
+ ownerId,
+ fileName: file.fileName,
+ requestedTitle: null,
+ contentHash: file.contentHash,
});
+ const upload = await supabase.storage
+ .from(env.SUPABASE_DOCUMENT_BUCKET)
+ .upload(storagePath, await readFile(file.absolutePath), {
+ contentType: "application/pdf",
+ upsert: false,
+ });
if (upload.error) throw new Error(upload.error.message);
const { error: documentError } = await supabase.from("documents").insert({
id: documentId,
owner_id: ownerId,
- title: titleFromFileName(file.fileName),
+ title: namePlan.title,
description: null,
file_name: file.fileName,
file_type: "application/pdf",
@@ -236,7 +254,7 @@ async function main() {
import_batch_id: batch.id,
status: "queued",
metadata: {
- source_title: titleFromFileName(file.fileName),
+ source_title: namePlan.title,
publisher: null,
jurisdiction: "Australia/WA",
version: null,
@@ -245,6 +263,12 @@ async function main() {
uploaded_at: new Date().toISOString(),
indexed_at: null,
uploaded_by: ownerId,
+ original_file_name: namePlan.originalFileName,
+ original_title: namePlan.originalTitle,
+ smart_title_base: namePlan.baseTitle,
+ smart_title_group_key: namePlan.duplicateGroupKey,
+ smart_title_duplicate_index: namePlan.duplicateIndex,
+ smart_title_duplicate_reason: namePlan.duplicateReason,
document_status: "unknown",
clinical_validation_status: "unverified",
extraction_quality: "unknown",
diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts
index 5f5e4f2f8..56e4d1913 100644
--- a/scripts/playwright-base-url.ts
+++ b/scripts/playwright-base-url.ts
@@ -1,9 +1,46 @@
import { execFileSync } from "node:child_process";
import path from "node:path";
+import { appName, localProjectId } from "./local-server-utils.mjs";
const projectRoot = path.resolve(__dirname, "..");
const ensureScript = path.join(projectRoot, "scripts", "ensure-local-server.mjs");
const localUrlPattern = /^http:\/\/localhost:\d+$/;
+const identityScript = `
+const http = require("node:http");
+const url = process.argv[1] + "/api/local-project-id";
+const request = http.get(url, { timeout: 1500 }, (response) => {
+ let body = "";
+ response.setEncoding("utf8");
+ response.on("data", (chunk) => { body += chunk; });
+ response.on("end", () => {
+ if (response.statusCode !== 200) process.exit(2);
+ process.stdout.write(body);
+ });
+});
+request.on("timeout", () => { request.destroy(); process.exit(3); });
+request.on("error", () => process.exit(4));
+`;
+
+function verifyLocalProjectIdentity(baseUrl: string) {
+ const output = execFileSync(process.execPath, ["-e", identityScript, baseUrl], {
+ cwd: projectRoot,
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "inherit"],
+ }).trim();
+ const payload = JSON.parse(output) as {
+ appName?: string;
+ projectId?: string;
+ localServer?: { safeLocalOrigin?: boolean };
+ };
+
+ if (
+ payload.appName !== appName ||
+ payload.projectId !== localProjectId(projectRoot) ||
+ payload.localServer?.safeLocalOrigin !== true
+ ) {
+ throw new Error(`Ensured URL failed /api/local-project-id guard: ${baseUrl}`);
+ }
+}
export function getPlaywrightBaseUrl() {
const output = execFileSync(process.execPath, [ensureScript, "--print-url"], {
@@ -16,5 +53,6 @@ export function getPlaywrightBaseUrl() {
throw new Error(`Expected ensure-local-server to print a localhost URL, received: ${output || ""}`);
}
+ verifyLocalProjectIdentity(output);
return output;
}
diff --git a/scripts/purge-query-logs.ts b/scripts/purge-query-logs.ts
new file mode 100644
index 000000000..fef3ce764
--- /dev/null
+++ b/scripts/purge-query-logs.ts
@@ -0,0 +1,68 @@
+import { loadEnvConfig } from "@next/env";
+import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils";
+
+loadEnvConfig(process.cwd());
+
+type PurgeArgs = {
+ ownerEmail?: string;
+ olderThanDays: number;
+ dryRun: boolean;
+};
+
+function parseArgs(argv: string[]): PurgeArgs {
+ const args: PurgeArgs = {
+ ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL,
+ olderThanDays: 90,
+ dryRun: false,
+ };
+
+ for (let index = 0; index < argv.length; index += 1) {
+ const token = argv[index];
+ if (token === "--dry-run") {
+ args.dryRun = true;
+ continue;
+ }
+ const value = argv[index + 1];
+ if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
+ index += 1;
+
+ if (token === "--owner-email") args.ownerEmail = value;
+ if (token === "--older-than-days") args.olderThanDays = Number.parseInt(value, 10);
+ }
+
+ if (!args.ownerEmail) throw new Error("Provide --owner-email.");
+ if (!Number.isInteger(args.olderThanDays) || args.olderThanDays <= 0) {
+ throw new Error("--older-than-days must be a positive integer.");
+ }
+ return args;
+}
+
+async function main() {
+ const args = parseArgs(process.argv.slice(2));
+ const supabase = await loadAdminClient();
+ const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail!);
+ const before = new Date(Date.now() - args.olderThanDays * 24 * 60 * 60 * 1000).toISOString();
+ const countQuery = supabase
+ .from("rag_queries")
+ .select("id", { count: "exact", head: true })
+ .eq("owner_id", ownerId)
+ .lt("created_at", before);
+ const { count, error: countError } = await countQuery;
+ if (countError) throw new Error(countError.message);
+
+ console.log(`RAG query logs older than ${args.olderThanDays} day(s): ${count ?? 0}`);
+ if (args.dryRun || !count) return;
+
+ const { error: deleteError } = await supabase
+ .from("rag_queries")
+ .delete()
+ .eq("owner_id", ownerId)
+ .lt("created_at", before);
+ if (deleteError) throw new Error(deleteError.message);
+ console.log(`Deleted ${count} old RAG query log(s).`);
+}
+
+main().catch((error) => {
+ console.error(error instanceof Error ? error.message : error);
+ process.exit(1);
+});
diff --git a/src/app/api/answer/route.ts b/src/app/api/answer/route.ts
index 6c77324bf..f35592b33 100644
--- a/src/app/api/answer/route.ts
+++ b/src/app/api/answer/route.ts
@@ -3,14 +3,15 @@ import { z } from "zod";
import { demoAnswer } from "@/lib/demo-data";
import { isDemoMode } from "@/lib/env";
import { answerQuestionWithScope } from "@/lib/rag";
-import { jsonError } from "@/lib/http";
-import { createAdminClient } from "@/lib/supabase/admin";
-import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
+import { jsonError, PublicApiError } from "@/lib/http";
+import { consumePublicAnswerRateLimit } from "@/lib/public-rate-limit";
+import { classifyRagQuery } from "@/lib/clinical-search";
+import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
export const runtime = "nodejs";
const answerSchema = z.object({
- query: z.string().trim().min(2),
+ query: z.string().trim().min(1),
documentId: z.string().uuid().optional(),
documentIds: z.array(z.string().uuid()).max(25).optional(),
});
@@ -19,25 +20,48 @@ export async function POST(request: Request) {
try {
const body = answerSchema.parse(await request.json());
if (isDemoMode()) {
+ const answer = demoAnswer(body.query, body.documentId, body.documentIds);
return NextResponse.json({
- ...demoAnswer(body.query, body.documentId, body.documentIds),
+ ...answer,
+ smartApiPlan: buildSmartRagApiPlan({
+ query: body.query,
+ queryClass: classifyRagQuery(body.query).queryClass,
+ results: answer.sources,
+ routeMode: answer.routingMode,
+ retrievalStrategy: "hybrid",
+ }),
demoMode: true,
});
}
- const supabase = createAdminClient();
- const user = await requireAuthenticatedUser(request, supabase);
+ const rateLimit = consumePublicAnswerRateLimit(request.headers);
+ if (rateLimit.limited) {
+ return NextResponse.json(
+ { error: "Too many public answer requests. Retry shortly." },
+ { status: 429, headers: { "Retry-After": String(rateLimit.retryAfterSeconds) } },
+ );
+ }
+
const answer = await answerQuestionWithScope({
query: body.query,
documentId: body.documentId,
documentIds: body.documentIds,
- ownerId: user.id,
+ ownerId: undefined,
});
return NextResponse.json(answer);
} catch (error) {
- if (error instanceof AuthenticationError) {
- return unauthorizedResponse();
+ if (error instanceof z.ZodError) {
+ return jsonError(error, 400);
+ }
+ if (error instanceof PublicApiError) {
+ return jsonError(error, error.status);
+ }
+ if (error instanceof Error) {
+ return jsonError(
+ new PublicApiError("Answer generation failed. Retry with a narrower question.", 500, { code: error.name }),
+ 500,
+ );
}
- return jsonError(error, 400);
+ return jsonError("Answer generation failed.", 500);
}
}
diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts
index 37eeec3c1..9b8ea37a5 100644
--- a/src/app/api/answer/stream/route.ts
+++ b/src/app/api/answer/stream/route.ts
@@ -2,14 +2,16 @@ import { z } from "zod";
import { demoAnswer } from "@/lib/demo-data";
import { isDemoMode } from "@/lib/env";
import { PublicApiError, jsonError } from "@/lib/http";
+import { consumePublicAnswerRateLimit, type PublicRateLimitResult } from "@/lib/public-rate-limit";
import { answerQuestionWithScope, type AnswerProgressEvent } from "@/lib/rag";
-import { createAdminClient } from "@/lib/supabase/admin";
-import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
+import { classifyRagQuery } from "@/lib/clinical-search";
+import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance";
+import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
export const runtime = "nodejs";
const answerSchema = z.object({
- query: z.string().trim().min(2),
+ query: z.string().trim().min(1),
documentId: z.string().uuid().optional(),
documentIds: z.array(z.string().uuid()).max(25).optional(),
});
@@ -20,6 +22,51 @@ function encodeSse(event: string, data: unknown) {
return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
}
+function rateLimitStream(rateLimit: PublicRateLimitResult) {
+ return new Response(
+ encodeSse("error", {
+ error: "Too many public answer requests. Retry shortly.",
+ status: 429,
+ details: { retryAfterSeconds: rateLimit.retryAfterSeconds, resetAt: rateLimit.resetAt },
+ }),
+ {
+ status: 429,
+ headers: {
+ "Content-Type": "text/event-stream; charset=utf-8",
+ "Cache-Control": "no-cache, no-transform",
+ "Retry-After": String(rateLimit.retryAfterSeconds),
+ },
+ },
+ );
+}
+
+function streamErrorPayload(error: unknown) {
+ if (error instanceof PublicApiError) {
+ return { message: error.message, status: error.status, details: error.details };
+ }
+
+ if (error instanceof Error) {
+ return {
+ message: "Answer generation failed. Retry with a narrower question.",
+ status: 503,
+ details: { code: error.name },
+ };
+ }
+
+ return {
+ message: "Search processing is temporarily unavailable.",
+ status: 503,
+ };
+}
+
+function logStreamError(error: unknown) {
+ console.error("Search stream failed", {
+ name: error instanceof Error ? error.name : typeof error,
+ message: error instanceof Error ? error.message : String(error),
+ stack: error instanceof Error ? error.stack : undefined,
+ });
+}
+
function streamAnswer(body: AnswerBody, ownerId?: string) {
const encoder = new TextEncoder();
@@ -34,7 +81,25 @@ function streamAnswer(body: AnswerBody, ownerId?: string) {
try {
send("progress", { stage: "retrieving", message: "Searching indexed documents." });
const answer = isDemoMode()
- ? { ...demoAnswer(body.query, body.documentId, body.documentIds), demoMode: true }
+ ? (() => {
+ const demo = demoAnswer(body.query, body.documentId, body.documentIds);
+ const sources = annotateSearchResults(body.query, demo.sources);
+ const relevance = buildEvidenceRelevance(body.query, sources);
+ return {
+ ...demo,
+ sources,
+ relevance,
+ smartPanel: demo.smartPanel ? { ...demo.smartPanel, relevance } : demo.smartPanel,
+ smartApiPlan: buildSmartRagApiPlan({
+ query: body.query,
+ queryClass: classifyRagQuery(body.query).queryClass,
+ results: sources,
+ routeMode: demo.routingMode,
+ retrievalStrategy: "hybrid",
+ }),
+ demoMode: true,
+ };
+ })()
: await answerQuestionWithScope({
query: body.query,
documentId: body.documentId,
@@ -44,11 +109,9 @@ function streamAnswer(body: AnswerBody, ownerId?: string) {
});
send("final", answer);
} catch (error) {
- const publicError =
- error instanceof PublicApiError
- ? error
- : new PublicApiError("Answer generation failed. Retry with a narrower question.", 500);
- send("error", { error: publicError.message });
+ logStreamError(error);
+ const streamError = streamErrorPayload(error);
+ send("error", { error: streamError.message, status: streamError.status, details: streamError.details });
} finally {
controller.close();
}
@@ -69,13 +132,20 @@ export async function POST(request: Request) {
const body = answerSchema.parse(await request.json());
if (isDemoMode()) return streamAnswer(body);
- const supabase = createAdminClient();
- const user = await requireAuthenticatedUser(request, supabase);
- return streamAnswer(body, user.id);
+ const rateLimit = consumePublicAnswerRateLimit(request.headers);
+ if (rateLimit.limited) return rateLimitStream(rateLimit);
+
+ return streamAnswer(body);
} catch (error) {
- if (error instanceof AuthenticationError) {
- return unauthorizedResponse();
+ if (error instanceof z.ZodError) {
+ return jsonError(error, 400);
+ }
+ if (error instanceof PublicApiError) {
+ return jsonError(error, error.status);
+ }
+ if (error instanceof Error) {
+ return jsonError(new PublicApiError("Answer processing failed.", 500, { code: error.name }), 500);
}
- return jsonError(error, 400);
+ return jsonError("Answer processing failed.", 500);
}
}
diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts
index 7575a4f03..181b8c83c 100644
--- a/src/app/api/documents/[id]/reindex/route.ts
+++ b/src/app/api/documents/[id]/reindex/route.ts
@@ -1,6 +1,7 @@
import { NextResponse } from "next/server";
import { env, isDemoMode } from "@/lib/env";
import { upsertDocumentEnrichment } from "@/lib/document-enrichment";
+import { upsertDocumentDeepMemory } from "@/lib/deep-memory";
import { jsonError } from "@/lib/http";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
@@ -27,7 +28,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
const { data: document, error: documentError } = await supabase
.from("documents")
- .select("id,owner_id,title,file_name,source_path,import_batch_id")
+ .select("id,owner_id,title,file_name,source_path,import_batch_id,metadata")
.eq("id", id)
.eq("owner_id", user.id)
.maybeSingle();
@@ -39,17 +40,17 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
const [chunksResult, imagesResult] = await Promise.all([
supabase
.from("document_chunks")
- .select("id,page_number,chunk_index,section_heading,content")
+ .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata")
.eq("document_id", id)
.order("chunk_index", { ascending: true })
- .limit(24),
+ .limit(1000),
supabase
.from("document_images")
- .select("id,page_number,caption,image_type,labels")
+ .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata")
.eq("document_id", id)
.eq("searchable", true)
.order("clinical_relevance_score", { ascending: false })
- .limit(12),
+ .limit(200),
]);
if (chunksResult.error) throw new Error(chunksResult.error.message);
@@ -64,7 +65,20 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
chunks: chunksResult.data,
images: imagesResult.data ?? [],
});
- return NextResponse.json({ mode, enrichment });
+ const deepMemory = await upsertDocumentDeepMemory({
+ supabase,
+ document,
+ chunks: chunksResult.data,
+ images: imagesResult.data ?? [],
+ });
+ return NextResponse.json({
+ mode,
+ enrichment,
+ deepMemory: {
+ sectionCount: deepMemory.sections.length,
+ memoryCardCount: deepMemory.memoryCards.length,
+ },
+ });
}
const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: id });
diff --git a/src/app/api/documents/[id]/route.ts b/src/app/api/documents/[id]/route.ts
index 05be91637..723d291a5 100644
--- a/src/app/api/documents/[id]/route.ts
+++ b/src/app/api/documents/[id]/route.ts
@@ -1,12 +1,163 @@
import { NextResponse } from "next/server";
+import { z } from "zod";
import { getDemoDocumentPayload } from "@/lib/demo-data";
-import { isDemoMode } from "@/lib/env";
-import { jsonError } from "@/lib/http";
+import { env, isDemoMode } from "@/lib/env";
+import { jsonError, PublicApiError } from "@/lib/http";
+import { invalidateRagCachesForOwner } from "@/lib/rag";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
export const runtime = "nodejs";
+const renameSchema = z.object({
+ title: z.string().trim().min(1).max(180),
+});
+
+function safeMetadata(value: unknown) {
+ return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {};
+}
+
+function metadataText(metadata: Record, key: string) {
+ const value = metadata[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function compactTableText(value: string | null, limit = 500) {
+ if (!value) return null;
+ const compact = value.replace(/\s+/g, " ").trim();
+ if (!compact) return null;
+ return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact;
+}
+
+function metadataStringArrayRows(metadata: Record, key: string) {
+ const value = metadata[key];
+ if (!Array.isArray(value)) return null;
+ const rows = value
+ .filter((row): row is unknown[] => Array.isArray(row))
+ .map((row) => row.map((cell) => String(cell ?? "").trim()));
+ return rows.length ? rows : null;
+}
+
+function metadataStringArray(metadata: Record, key: string) {
+ const value = metadata[key];
+ if (!Array.isArray(value)) return null;
+ const items = value.map((item) => String(item ?? "").trim()).filter(Boolean);
+ return items.length ? items : null;
+}
+
+function withImageTableMetadata(image: T) {
+ const metadata = safeMetadata(image.metadata);
+ const rawTableText = metadataText(metadata, "table_text");
+ const tableText = rawTableText ?? metadataText(metadata, "table_text_snippet");
+ const publicImage = { ...image };
+ delete publicImage.metadata;
+ return {
+ ...publicImage,
+ tableLabel: metadataText(metadata, "table_label"),
+ tableTitle: metadataText(metadata, "table_title"),
+ tableRole: metadataText(metadata, "table_role"),
+ tableTextSnippet: compactTableText(tableText),
+ clinicalUseClass: metadataText(metadata, "clinical_use_class"),
+ clinicalUseReason: metadataText(metadata, "clinical_use_reason"),
+ accessibleTableMarkdown: metadataText(metadata, "accessible_table_markdown") ?? rawTableText,
+ tableRows: metadataStringArrayRows(metadata, "table_rows"),
+ tableColumns: metadataStringArray(metadata, "table_columns"),
+ };
+}
+
+function storageWarningsFrom(error: unknown, label: string) {
+ const message =
+ error && typeof error === "object" && "message" in error ? String(error.message) : "Storage cleanup failed.";
+ return `${label}: ${message}`;
+}
+
+async function removeStorageObjects(args: {
+ supabase: ReturnType;
+ sourcePath: string | null;
+ imagePaths: string[];
+}) {
+ const warnings: string[] = [];
+ let storageRemoved = 0;
+
+ if (args.sourcePath) {
+ const sourceRemove = await args.supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([args.sourcePath]);
+ if (sourceRemove.error) {
+ warnings.push(storageWarningsFrom(sourceRemove.error, "Source PDF"));
+ } else {
+ storageRemoved += sourceRemove.data?.length ?? 0;
+ }
+ }
+
+ const uniqueImagePaths = Array.from(new Set(args.imagePaths.filter(Boolean)));
+ for (let start = 0; start < uniqueImagePaths.length; start += 1000) {
+ const paths = uniqueImagePaths.slice(start, start + 1000);
+ const imageRemove = await args.supabase.storage.from(env.SUPABASE_IMAGE_BUCKET).remove(paths);
+ if (imageRemove.error) {
+ warnings.push(storageWarningsFrom(imageRemove.error, "Extracted images"));
+ } else {
+ storageRemoved += imageRemove.data?.length ?? 0;
+ }
+ }
+
+ return { storageRemoved, storageWarnings: warnings };
+}
+
+async function createStorageCleanupJob(args: {
+ supabase: ReturnType;
+ ownerId: string;
+ documentId: string;
+ documentTitle: string;
+ sourcePath: string | null;
+ imagePaths: string[];
+}) {
+ const { data, error } = await args.supabase
+ .from("storage_cleanup_jobs")
+ .insert({
+ owner_id: args.ownerId,
+ document_id: args.documentId,
+ document_title: args.documentTitle,
+ document_bucket: env.SUPABASE_DOCUMENT_BUCKET,
+ document_paths: args.sourcePath ? [args.sourcePath] : [],
+ image_bucket: env.SUPABASE_IMAGE_BUCKET,
+ image_paths: Array.from(new Set(args.imagePaths.filter(Boolean))),
+ status: "pending",
+ metadata: {
+ operation: "permanent_document_delete",
+ created_by: "api/documents/[id]",
+ },
+ })
+ .select("id")
+ .single();
+
+ if (error) throw new Error(error.message);
+ return data.id as string;
+}
+
+async function updateStorageCleanupJob(args: {
+ supabase: ReturnType;
+ cleanupJobId: string;
+ status: "completed" | "failed";
+ storageRemoved: number;
+ warnings: string[];
+}) {
+ const { error } = await args.supabase
+ .from("storage_cleanup_jobs")
+ .update({
+ status: args.status,
+ attempts: 1,
+ storage_removed: args.storageRemoved,
+ last_error: args.warnings.length ? args.warnings.join("; ") : null,
+ completed_at: args.status === "completed" ? new Date().toISOString() : null,
+ metadata: {
+ operation: "permanent_document_delete",
+ storage_warnings: args.warnings,
+ },
+ })
+ .eq("id", args.cleanupJobId);
+
+ return error ? storageWarningsFrom(error, "Cleanup ledger") : null;
+}
+
export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) {
try {
const { id } = await params;
@@ -43,10 +194,11 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:
const { data: images, error: imagesError } = await supabase
.from("document_images")
.select(
- "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels",
+ "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata",
)
.eq("document_id", id)
- .eq("searchable", true)
+ .neq("image_type", "logo_decorative")
+ .or("searchable.eq.true,source_kind.eq.table_crop")
.order("page_number", { ascending: true });
if (imagesError) throw new Error(imagesError.message);
@@ -79,7 +231,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:
summary: summaryResult.data ?? null,
},
pages: pages ?? [],
- images: images ?? [],
+ images: (images ?? []).map(withImageTableMetadata),
chunks: chunks ?? [],
});
} catch (error) {
@@ -89,3 +241,167 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:
return jsonError(error);
}
}
+
+export async function PATCH(request: Request, { params }: { params: Promise<{ id: string }> }) {
+ try {
+ const { id } = await params;
+ if (isDemoMode()) {
+ return NextResponse.json({ error: "Demo documents cannot be renamed." }, { status: 400 });
+ }
+
+ const parsed = renameSchema.safeParse(await request.json().catch(() => null));
+ if (!parsed.success) {
+ throw new PublicApiError("Enter a document title between 1 and 180 characters.");
+ }
+
+ const supabase = createAdminClient();
+ const user = await requireAuthenticatedUser(request, supabase);
+ const { data: document, error: documentError } = await supabase
+ .from("documents")
+ .select("id,owner_id,title,file_name,storage_path,content_hash,metadata")
+ .eq("id", id)
+ .eq("owner_id", user.id)
+ .maybeSingle();
+
+ if (documentError) throw new Error(documentError.message);
+ if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 });
+ if (document.title.trim() === parsed.data.title) {
+ throw new PublicApiError("Document title is unchanged.");
+ }
+
+ const metadata = safeMetadata(document.metadata);
+ const { data: updated, error: updateError } = await supabase
+ .from("documents")
+ .update({
+ title: parsed.data.title,
+ metadata: {
+ ...metadata,
+ renamed_at: new Date().toISOString(),
+ previous_title: document.title,
+ original_file_name: metadata.original_file_name ?? document.file_name,
+ },
+ })
+ .eq("id", id)
+ .eq("owner_id", user.id)
+ .select("*")
+ .single();
+
+ if (updateError) throw new Error(updateError.message);
+ invalidateRagCachesForOwner(user.id);
+ return NextResponse.json({ document: updated });
+ } catch (error) {
+ if (error instanceof AuthenticationError) {
+ return unauthorizedResponse();
+ }
+ return jsonError(error);
+ }
+}
+
+export async function DELETE(request: Request, { params }: { params: Promise<{ id: string }> }) {
+ try {
+ const { id } = await params;
+ if (isDemoMode()) {
+ return NextResponse.json({ error: "Demo documents cannot be deleted." }, { status: 400 });
+ }
+
+ const supabase = createAdminClient();
+ const user = await requireAuthenticatedUser(request, supabase);
+ const { data: document, error: documentError } = await supabase
+ .from("documents")
+ .select("id,owner_id,title,storage_path")
+ .eq("id", id)
+ .eq("owner_id", user.id)
+ .maybeSingle();
+
+ if (documentError) throw new Error(documentError.message);
+ if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 });
+
+ const { data: activeJobs, error: activeJobsError } = await supabase
+ .from("ingestion_jobs")
+ .select("id,status")
+ .eq("document_id", id)
+ .eq("status", "processing")
+ .limit(1);
+
+ if (activeJobsError) throw new Error(activeJobsError.message);
+ if ((activeJobs ?? []).length > 0) {
+ throw new PublicApiError("Document is currently indexing. Stop or wait for the worker before deleting.", 409);
+ }
+
+ const [{ data: images, error: imagesError }, { data: chunks, error: chunksError }] = await Promise.all([
+ supabase.from("document_images").select("storage_path").eq("document_id", id),
+ supabase.from("document_chunks").select("id").eq("document_id", id),
+ ]);
+
+ if (imagesError) throw new Error(imagesError.message);
+ if (chunksError) throw new Error(chunksError.message);
+
+ const chunkIds = (chunks ?? []).map((chunk) => chunk.id).filter(Boolean);
+ const imagePaths = (images ?? []).map((image) => image.storage_path).filter(Boolean);
+ const cleanupJobId = await createStorageCleanupJob({
+ supabase,
+ ownerId: user.id,
+ documentId: id,
+ documentTitle: document.title,
+ sourcePath: document.storage_path,
+ imagePaths,
+ });
+
+ if (chunkIds.length > 0) {
+ const { error: queryDeleteError } = await supabase
+ .from("rag_queries")
+ .delete()
+ .eq("owner_id", user.id)
+ .overlaps("source_chunk_ids", chunkIds);
+ if (queryDeleteError) {
+ const ledgerWarning = await updateStorageCleanupJob({
+ supabase,
+ cleanupJobId,
+ status: "failed",
+ storageRemoved: 0,
+ warnings: [`Query log delete: ${queryDeleteError.message}`],
+ });
+ throw new Error(ledgerWarning ? `${queryDeleteError.message}; ${ledgerWarning}` : queryDeleteError.message);
+ }
+ }
+
+ const { error: deleteError } = await supabase.from("documents").delete().eq("id", id).eq("owner_id", user.id);
+ if (deleteError) {
+ const ledgerWarning = await updateStorageCleanupJob({
+ supabase,
+ cleanupJobId,
+ status: "failed",
+ storageRemoved: 0,
+ warnings: [`Database delete: ${deleteError.message}`],
+ });
+ throw new Error(ledgerWarning ? `${deleteError.message}; ${ledgerWarning}` : deleteError.message);
+ }
+
+ const cleanup = await removeStorageObjects({
+ supabase,
+ sourcePath: document.storage_path,
+ imagePaths,
+ });
+ const ledgerWarning = await updateStorageCleanupJob({
+ supabase,
+ cleanupJobId,
+ status: cleanup.storageWarnings.length > 0 ? "failed" : "completed",
+ storageRemoved: cleanup.storageRemoved,
+ warnings: cleanup.storageWarnings,
+ });
+ if (ledgerWarning) cleanup.storageWarnings.push(ledgerWarning);
+
+ invalidateRagCachesForOwner(user.id);
+ return NextResponse.json({
+ deleted: true,
+ documentId: id,
+ storageRemoved: cleanup.storageRemoved,
+ storageWarnings: cleanup.storageWarnings,
+ });
+ } catch (error) {
+ if (error instanceof AuthenticationError) {
+ return unauthorizedResponse();
+ }
+ return jsonError(error);
+ }
+}
diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts
index c88fb2898..95ce5eb21 100644
--- a/src/app/api/documents/route.ts
+++ b/src/app/api/documents/route.ts
@@ -7,49 +7,189 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f
export const runtime = "nodejs";
+const DOCUMENT_LIST_COLUMNS = [
+ "id",
+ "owner_id",
+ "title",
+ "description",
+ "file_name",
+ "file_type",
+ "file_size",
+ "storage_path",
+ "content_hash",
+ "source_path",
+ "import_batch_id",
+ "status",
+ "page_count",
+ "chunk_count",
+ "image_count",
+ "error_message",
+ "metadata",
+ "created_at",
+ "updated_at",
+].join(",");
+
+const LABEL_LIST_COLUMNS = [
+ "id",
+ "document_id",
+ "owner_id",
+ "label",
+ "label_type",
+ "source",
+ "confidence",
+ "metadata",
+ "created_at",
+ "updated_at",
+].join(",");
+
+const SUMMARY_LIST_COLUMNS = [
+ "id",
+ "document_id",
+ "owner_id",
+ "summary",
+ "clinical_specifics",
+ "source_chunk_ids",
+ "source_image_ids",
+ "model",
+ "metadata",
+ "generated_at",
+ "created_at",
+ "updated_at",
+].join(",");
+
+const VALID_STATUSES = new Set(["queued", "processing", "indexed", "failed"]);
+const ACTIVE_DOCUMENT_STATUSES = new Set(["queued", "processing"]);
+const ACTIVE_INDEXING_POLL_MS = 5_000;
+
+type DocumentListRow = Record & { id: string; status?: string | null };
+type LabelListRow = Record & { document_id: string };
+type SummaryListRow = Record & { document_id: string };
+
+function parsePositiveInt(value: string | null, fallback: number, max: number) {
+ const parsed = Number.parseInt(value ?? "", 10);
+ if (!Number.isInteger(parsed) || parsed <= 0) return fallback;
+ return Math.min(parsed, max);
+}
+
+function parseOffset(value: string | null) {
+ const parsed = Number.parseInt(value ?? "", 10);
+ return Number.isInteger(parsed) && parsed > 0 ? parsed : 0;
+}
+
+function ilikePattern(value: string) {
+ return `%${value.replace(/[%_]/g, "\\$&")}%`;
+}
+
+function safeSearchTerm(value: string) {
+ return value
+ .replace(/[,%()]/g, " ")
+ .replace(/\s+/g, " ")
+ .trim()
+ .slice(0, 120);
+}
+
+function indexingState(documents: DocumentListRow[]) {
+ const active = documents.some((document) => ACTIVE_DOCUMENT_STATUSES.has(String(document.status ?? "")));
+ return {
+ active,
+ pollAfterMs: active ? ACTIVE_INDEXING_POLL_MS : null,
+ };
+}
+
+function documentsResponse(payload: Record, indexing: ReturnType) {
+ return NextResponse.json(
+ {
+ ...payload,
+ indexing,
+ },
+ {
+ headers: {
+ "Cache-Control": "private, no-store",
+ "X-Indexing-Active": String(indexing.active),
+ "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""),
+ },
+ },
+ );
+}
+
export async function GET(request: Request) {
try {
if (isDemoMode()) {
- return NextResponse.json({ documents: demoDocuments, demoMode: true });
+ return documentsResponse({ documents: demoDocuments, demoMode: true }, { active: false, pollAfterMs: null });
}
+ const url = new URL(request.url);
+ const limit = parsePositiveInt(url.searchParams.get("limit"), 100, 200);
+ const offset = parseOffset(url.searchParams.get("offset"));
+ const search = safeSearchTerm(url.searchParams.get("q") ?? "");
+ const status = url.searchParams.get("status")?.trim() ?? "";
+ const includeMeta = url.searchParams.get("includeMeta") !== "false";
+
const supabase = createAdminClient();
const user = await requireAuthenticatedUser(request, supabase);
- const { data, error } = await supabase
+ let query = supabase
.from("documents")
- .select("*")
+ .select(DOCUMENT_LIST_COLUMNS, { count: "exact" })
.eq("owner_id", user.id)
- .order("created_at", { ascending: false });
+ .order("created_at", { ascending: false })
+ .range(offset, offset + limit - 1);
+
+ if (status && VALID_STATUSES.has(status)) {
+ query = query.eq("status", status);
+ }
+ if (search) {
+ const pattern = ilikePattern(search);
+ query = query.or(`title.ilike.${pattern},file_name.ilike.${pattern}`);
+ }
+
+ const { data, error, count } = await query;
if (error) throw new Error(error.message);
- const documents = data ?? [];
+ const documents = (data ?? []) as unknown as DocumentListRow[];
const documentIds = documents.map((document) => document.id);
+ const indexing = indexingState(documents);
- if (documentIds.length === 0) return NextResponse.json({ documents });
+ const pagination = {
+ limit,
+ offset,
+ total: count ?? documents.length,
+ nextOffset: offset + documents.length,
+ hasMore: count === null ? documents.length === limit : offset + documents.length < count,
+ };
+
+ if (documentIds.length === 0 || !includeMeta) {
+ return documentsResponse({ documents, pagination }, indexing);
+ }
const [labelsResult, summariesResult] = await Promise.all([
- supabase.from("document_labels").select("*").in("document_id", documentIds),
- supabase.from("document_summaries").select("*").in("document_id", documentIds),
+ supabase.from("document_labels").select(LABEL_LIST_COLUMNS).in("document_id", documentIds),
+ supabase.from("document_summaries").select(SUMMARY_LIST_COLUMNS).in("document_id", documentIds),
]);
if (labelsResult.error) throw new Error(labelsResult.error.message);
if (summariesResult.error) throw new Error(summariesResult.error.message);
const labelsByDocument = new Map();
- for (const label of labelsResult.data ?? []) {
+ for (const label of (labelsResult.data ?? []) as unknown as LabelListRow[]) {
const existing = labelsByDocument.get(label.document_id) ?? [];
existing.push(label);
labelsByDocument.set(label.document_id, existing);
}
- const summariesByDocument = new Map((summariesResult.data ?? []).map((summary) => [summary.document_id, summary]));
-
- return NextResponse.json({
- documents: documents.map((document) => ({
- ...document,
- labels: labelsByDocument.get(document.id) ?? [],
- summary: summariesByDocument.get(document.id) ?? null,
- })),
- });
+ const summariesByDocument = new Map(
+ ((summariesResult.data ?? []) as unknown as SummaryListRow[]).map((summary) => [summary.document_id, summary]),
+ );
+
+ return documentsResponse(
+ {
+ documents: documents.map((document) => ({
+ ...document,
+ labels: labelsByDocument.get(document.id) ?? [],
+ summary: summariesByDocument.get(document.id) ?? null,
+ })),
+ pagination,
+ },
+ indexing,
+ );
} catch (error) {
if (error instanceof AuthenticationError) {
return unauthorizedResponse();
diff --git a/src/app/api/ingestion/batches/route.ts b/src/app/api/ingestion/batches/route.ts
index 4d2a2b1e0..a57c3386c 100644
--- a/src/app/api/ingestion/batches/route.ts
+++ b/src/app/api/ingestion/batches/route.ts
@@ -6,9 +6,41 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f
export const runtime = "nodejs";
+const ACTIVE_BATCH_STATUSES = new Set(["queued", "processing"]);
+const ACTIVE_INDEXING_POLL_MS = 5_000;
+
+type BatchRow = Record & { status?: string | null };
+
+function batchesIndexingState(batches: BatchRow[]) {
+ const activeBatchCount = batches.filter((batch) => ACTIVE_BATCH_STATUSES.has(String(batch.status ?? ""))).length;
+ return {
+ activeBatchCount,
+ hasActiveBatches: activeBatchCount > 0,
+ pollAfterMs: activeBatchCount > 0 ? ACTIVE_INDEXING_POLL_MS : null,
+ };
+}
+
+function batchesResponse(batches: BatchRow[], extra: Record = {}) {
+ const indexing = batchesIndexingState(batches);
+ return NextResponse.json(
+ {
+ batches,
+ ...indexing,
+ ...extra,
+ },
+ {
+ headers: {
+ "Cache-Control": "private, no-store",
+ "X-Indexing-Active": String(indexing.hasActiveBatches),
+ "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""),
+ },
+ },
+ );
+}
+
export async function GET(request: Request) {
try {
- if (isDemoMode()) return NextResponse.json({ batches: [], demoMode: true });
+ if (isDemoMode()) return batchesResponse([], { demoMode: true });
const supabase = createAdminClient();
const user = await requireAuthenticatedUser(request, supabase);
@@ -20,7 +52,7 @@ export async function GET(request: Request) {
.limit(20);
if (error) throw new Error(error.message);
- return NextResponse.json({ batches: data ?? [] });
+ return batchesResponse((data ?? []) as unknown as BatchRow[]);
} catch (error) {
if (error instanceof AuthenticationError) return unauthorizedResponse();
return jsonError(error);
diff --git a/src/app/api/ingestion/jobs/route.ts b/src/app/api/ingestion/jobs/route.ts
index c7e4fd85e..6a9ab2064 100644
--- a/src/app/api/ingestion/jobs/route.ts
+++ b/src/app/api/ingestion/jobs/route.ts
@@ -6,9 +6,41 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f
export const runtime = "nodejs";
+const ACTIVE_JOB_STATUSES = new Set(["pending", "processing"]);
+const ACTIVE_INDEXING_POLL_MS = 5_000;
+
+type JobRow = Record & { status?: string | null };
+
+function jobsIndexingState(jobs: JobRow[]) {
+ const activeJobCount = jobs.filter((job) => ACTIVE_JOB_STATUSES.has(String(job.status ?? ""))).length;
+ return {
+ activeJobCount,
+ hasActiveJobs: activeJobCount > 0,
+ pollAfterMs: activeJobCount > 0 ? ACTIVE_INDEXING_POLL_MS : null,
+ };
+}
+
+function jobsResponse(jobs: JobRow[], extra: Record = {}) {
+ const indexing = jobsIndexingState(jobs);
+ return NextResponse.json(
+ {
+ jobs,
+ ...indexing,
+ ...extra,
+ },
+ {
+ headers: {
+ "Cache-Control": "private, no-store",
+ "X-Indexing-Active": String(indexing.hasActiveJobs),
+ "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""),
+ },
+ },
+ );
+}
+
export async function GET(request: Request) {
try {
- if (isDemoMode()) return NextResponse.json({ jobs: [], demoMode: true });
+ if (isDemoMode()) return jobsResponse([], { demoMode: true });
const supabase = createAdminClient();
const user = await requireAuthenticatedUser(request, supabase);
@@ -25,7 +57,7 @@ export async function GET(request: Request) {
const { data, error } = await query;
if (error) throw new Error(error.message);
- return NextResponse.json({ jobs: data ?? [] });
+ return jobsResponse((data ?? []) as unknown as JobRow[]);
} catch (error) {
if (error instanceof AuthenticationError) return unauthorizedResponse();
return jsonError(error);
diff --git a/src/app/api/local-project-id/route.ts b/src/app/api/local-project-id/route.ts
index 57316373f..555742d0e 100644
--- a/src/app/api/local-project-id/route.ts
+++ b/src/app/api/local-project-id/route.ts
@@ -1,11 +1,12 @@
-import { appName, localProjectId } from "../../../../scripts/local-server-utils.mjs";
+import { localProjectRequestIdentityPayload } from "@/lib/local-project-guard";
export const dynamic = "force-dynamic";
export const runtime = "nodejs";
-export async function GET() {
- return Response.json({
- appName,
- projectId: localProjectId(process.cwd()),
+export async function GET(request: Request) {
+ return Response.json(localProjectRequestIdentityPayload(request), {
+ headers: {
+ "Cache-Control": "no-store",
+ },
});
}
diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts
index 87aefd48d..5e1c152e3 100644
--- a/src/app/api/search/route.ts
+++ b/src/app/api/search/route.ts
@@ -3,77 +3,249 @@ import { z } from "zod";
import { demoSearch } from "@/lib/demo-data";
import { isDemoMode } from "@/lib/env";
import { buildSmartPanel, buildVisualEvidence, diversifySearchResults } from "@/lib/evidence";
-import { fetchRelatedDocuments } from "@/lib/document-enrichment";
-import { jsonError } from "@/lib/http";
+import { annotateDocumentMatches, annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance";
+import { fetchRelatedDocuments, toDocumentMatch } from "@/lib/document-enrichment";
+import { jsonError, PublicApiError } from "@/lib/http";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
import { searchChunksWithTelemetry } from "@/lib/rag";
+import { classifyRagQuery } from "@/lib/clinical-search";
+import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
import { createAdminClient } from "@/lib/supabase/admin";
-import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
+import { consumePublicSearchRateLimit } from "@/lib/public-rate-limit";
+import type { SearchResult } from "@/lib/types";
export const runtime = "nodejs";
const searchSchema = z.object({
- query: z.string().trim().min(2),
+ query: z.string().trim().min(1),
topK: z.number().int().min(1).max(20).optional(),
documentId: z.string().uuid().optional(),
documentIds: z.array(z.string().uuid()).max(25).optional(),
+ mode: z.enum(["answer", "documents"]).optional().default("answer"),
+ documentLimit: z.number().int().min(1).max(50).optional().default(20),
includeRelatedDocuments: z.boolean().optional().default(true),
});
+type SearchRequestBody = z.infer;
+
+const publicSearchInflight = new Map>();
+
+function publicSearchKey(body: SearchRequestBody) {
+ return JSON.stringify({
+ query: body.query.toLowerCase().replace(/\s+/g, " ").trim(),
+ topK: body.topK ?? null,
+ documentId: body.documentId ?? null,
+ documentIds: body.documentIds?.length ? [...body.documentIds].sort() : [],
+ mode: body.mode,
+ documentLimit: body.documentLimit,
+ includeRelatedDocuments: body.includeRelatedDocuments,
+ });
+}
+
+async function coalescePublicSearch>(key: string, producer: () => Promise) {
+ const existing = publicSearchInflight.get(key) as Promise | undefined;
+ if (existing) return { payload: await existing, coalesced: true };
+
+ const pending = producer().finally(() => {
+ publicSearchInflight.delete(key);
+ });
+ publicSearchInflight.set(key, pending);
+ return { payload: await pending, coalesced: false };
+}
+
+function buildDocumentMatchesFromResults(results: SearchResult[], limit: number) {
+ const grouped = new Map<
+ string,
+ {
+ document_id: string;
+ title: string;
+ file_name: string;
+ bestPages: number[];
+ bestChunkIds: string[];
+ imageCount: number;
+ tableCount: number;
+ score: number;
+ }
+ >();
+ for (const result of results) {
+ const current = grouped.get(result.document_id);
+ const score = result.hybrid_score ?? result.similarity;
+ const page = result.page_number ?? null;
+ const clinicalImages = result.images?.filter((image) => isClinicalImageEvidence(image)) ?? [];
+ const tableCount = clinicalImages.filter((image) => image.source_kind === "table_crop").length;
+ const imageCount = clinicalImages.length;
+ if (!current) {
+ grouped.set(result.document_id, {
+ document_id: result.document_id,
+ title: result.title,
+ file_name: result.file_name,
+ bestPages: page ? [page] : [],
+ bestChunkIds: [result.id],
+ imageCount,
+ tableCount,
+ score,
+ });
+ continue;
+ }
+ current.score = Math.max(current.score, score);
+ if (page && !current.bestPages.includes(page)) current.bestPages.push(page);
+ if (!current.bestChunkIds.includes(result.id)) current.bestChunkIds.push(result.id);
+ current.imageCount += imageCount;
+ current.tableCount += tableCount;
+ }
+
+ return Array.from(grouped.values())
+ .sort((a, b) => b.score - a.score)
+ .slice(0, limit)
+ .map((document) => ({
+ ...document,
+ labels: [],
+ summarySnippet: null,
+ matchReason: `Matched ${document.bestChunkIds.length} indexed passage${
+ document.bestChunkIds.length === 1 ? "" : "s"
+ }`,
+ }));
+}
+
+async function buildPublicSearchPayload(body: SearchRequestBody) {
+ const supabase = createAdminClient();
+ const search = await searchChunksWithTelemetry({
+ query: body.query,
+ topK: body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8),
+ documentId: body.documentId,
+ documentIds: body.documentIds,
+ ownerId: undefined,
+ });
+ const resultLimit =
+ body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8);
+ const results = annotateSearchResults(body.query, diversifySearchResults(search.results, resultLimit, 4, true));
+
+ const relatedDocuments = body.includeRelatedDocuments
+ ? await fetchRelatedDocuments({
+ supabase,
+ ownerId: undefined,
+ query: body.query,
+ results,
+ limit: body.mode === "documents" ? body.documentLimit : undefined,
+ })
+ : [];
+ const smartPanel = buildSmartPanel(body.query, results);
+ const relevance = buildEvidenceRelevance(body.query, results);
+ const documentMatches =
+ body.mode === "documents"
+ ? annotateDocumentMatches(body.query, relatedDocuments.map(toDocumentMatch), results)
+ : [];
+ const queryClass = search.telemetry.query_class ?? classifyRagQuery(body.query).queryClass;
+ const smartApiPlan = buildSmartRagApiPlan({
+ query: body.query,
+ queryClass,
+ results,
+ retrievalStrategy: search.telemetry.retrieval_strategy,
+ routeMode: body.mode === "documents" ? undefined : "fast",
+ preferredResponseMode: body.mode === "documents" ? "document_lookup" : undefined,
+ });
+
+ return {
+ results,
+ visualEvidence: buildVisualEvidence(results),
+ relevance,
+ relatedDocuments,
+ documentMatches,
+ smartPanel: { ...smartPanel, relevance, relatedDocuments },
+ smartApiPlan,
+ telemetry: {
+ query_class: queryClass,
+ relevance_verdict: relevance.verdict,
+ relevance_score: relevance.score,
+ direct_source_count: relevance.directSourceCount,
+ weak_source_count: relevance.weakSourceCount,
+ retrieval_strategy: search.telemetry.retrieval_strategy,
+ smart_api_intent: smartApiPlan.intent,
+ smart_api_response_mode: smartApiPlan.responseMode,
+ smart_api_source_link_count: smartApiPlan.sourceLinkCount,
+ search_cache_hit: search.telemetry.search_cache_hit,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ memory_card_count: search.telemetry.memory_card_count,
+ memory_top_score: search.telemetry.memory_top_score,
+ weighted_top_score: search.telemetry.weighted_top_score,
+ rrf_top_score: search.telemetry.rrf_top_score,
+ },
+ };
+}
+
export async function POST(request: Request) {
try {
const body = searchSchema.parse(await request.json());
if (isDemoMode()) {
- const results = demoSearch(body.query, body.topK ?? 8, body.documentId, body.documentIds);
+ const results = annotateSearchResults(
+ body.query,
+ demoSearch(body.query, body.topK ?? 8, body.documentId, body.documentIds),
+ );
+ const queryClass = classifyRagQuery(body.query).queryClass;
+ const relevance = buildEvidenceRelevance(body.query, results);
+ const documentMatches =
+ body.mode === "documents"
+ ? annotateDocumentMatches(body.query, buildDocumentMatchesFromResults(results, body.documentLimit), results)
+ : [];
return NextResponse.json({
results,
visualEvidence: buildVisualEvidence(results),
- smartPanel: buildSmartPanel(body.query, results),
+ relevance,
+ smartPanel: { ...buildSmartPanel(body.query, results), relevance },
+ smartApiPlan: buildSmartRagApiPlan({
+ query: body.query,
+ queryClass,
+ results,
+ retrievalStrategy: "hybrid",
+ routeMode: body.mode === "documents" ? undefined : "fast",
+ preferredResponseMode: body.mode === "documents" ? "document_lookup" : undefined,
+ }),
relatedDocuments: [],
+ documentMatches,
demoMode: true,
});
}
- const supabase = createAdminClient();
- const user = await requireAuthenticatedUser(request, supabase);
- const search = await searchChunksWithTelemetry({
- query: body.query,
- topK: body.topK ?? 8,
- documentId: body.documentId,
- documentIds: body.documentIds,
- ownerId: user.id,
- });
- const results = diversifySearchResults(search.results, body.topK ?? 8);
-
- const relatedDocuments = body.includeRelatedDocuments
- ? await fetchRelatedDocuments({
- supabase,
- ownerId: user.id,
- query: body.query,
- results,
- })
- : [];
- const smartPanel = buildSmartPanel(body.query, results);
+ const rateLimit = consumePublicSearchRateLimit(request.headers);
+ if (rateLimit.limited) {
+ return NextResponse.json(
+ {
+ error: "Search is temporarily rate limited because too many requests were received. Retry shortly.",
+ retryAfterSeconds: rateLimit.retryAfterSeconds,
+ },
+ {
+ status: 429,
+ headers: {
+ "Retry-After": String(rateLimit.retryAfterSeconds),
+ },
+ },
+ );
+ }
+ const key = publicSearchKey(body);
+ const { payload, coalesced } = await coalescePublicSearch(key, () => buildPublicSearchPayload(body));
return NextResponse.json({
- results,
- visualEvidence: buildVisualEvidence(results),
- relatedDocuments,
- smartPanel: { ...smartPanel, relatedDocuments },
+ ...payload,
telemetry: {
- retrieval_strategy: search.telemetry.retrieval_strategy,
- search_cache_hit: search.telemetry.search_cache_hit,
- embedding_skipped: search.telemetry.embedding_skipped,
- embedding_cache_hit: search.telemetry.embedding_cache_hit,
- text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
- embedding_latency_ms: search.telemetry.embedding_latency_ms,
- supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
- rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ ...payload.telemetry,
+ coalesced,
},
});
} catch (error) {
- if (error instanceof AuthenticationError) {
- return unauthorizedResponse();
+ if (error instanceof z.ZodError) {
+ return jsonError(error, 400);
+ }
+ if (error instanceof Error && error.message.trim()) {
+ return jsonError(
+ new PublicApiError("Search failed. Retry with a narrower question.", 500, { code: error.name }),
+ 500,
+ );
}
- return jsonError(error, 400);
+ return jsonError(error, 500);
}
}
diff --git a/src/app/api/setup-status/route.ts b/src/app/api/setup-status/route.ts
index 02b855f2d..3ab615dd1 100644
--- a/src/app/api/setup-status/route.ts
+++ b/src/app/api/setup-status/route.ts
@@ -1,5 +1,6 @@
import { NextResponse } from "next/server";
import { env, isDemoMode } from "@/lib/env";
+import { localProjectRequestIdentityPayload, unsafeLocalProjectResponse } from "@/lib/local-project-guard";
import { createAdminClient } from "@/lib/supabase/admin";
import { checkSupabaseProjectConfig, formatSupabaseProjectCheck } from "@/lib/supabase/project";
@@ -16,15 +17,37 @@ type SetupCheck = {
detail: string;
};
+type WorkerStatus = {
+ check: SetupCheck;
+ activeWork: boolean;
+};
+
+type SetupStatusPayload = {
+ demoMode: boolean;
+ checks: SetupCheck[];
+ indexingActive: boolean;
+ pollAfterMs: number | null;
+ generatedAt: string;
+};
+
+type AdminClient = ReturnType;
+
const requiredSupabaseEnvPresent = Boolean(env.NEXT_PUBLIC_SUPABASE_URL && env.SUPABASE_SERVICE_ROLE_KEY);
const supabaseProjectCheck = checkSupabaseProjectConfig(env, { requireMetadata: true });
const supabaseProjectCanBeQueried = requiredSupabaseEnvPresent && supabaseProjectCheck.status !== "mismatch";
+const ACTIVE_INDEXING_POLL_MS = Math.max(3_000, Math.min(env.WORKER_POLL_MS, 15_000));
+const SETUP_RECHECK_POLL_MS = 60_000;
+const SETUP_STATUS_ACTIVE_CACHE_MS = Math.max(2_000, Math.min(ACTIVE_INDEXING_POLL_MS, 5_000));
+const SETUP_STATUS_IDLE_CACHE_MS = 30_000;
+
+let setupStatusCache: { expiresAt: number; payload: SetupStatusPayload } | null = null;
+let setupStatusInFlight: Promise | null = null;
function check(id: SetupCheckId, label: string, status: SetupCheckStatus, detail: string): SetupCheck {
return { id, label, status, detail };
}
-async function readSchemaStatus() {
+async function readSchemaStatus(supabase: AdminClient | null) {
if (!requiredSupabaseEnvPresent) {
return check(
"schema",
@@ -44,7 +67,9 @@ async function readSchemaStatus() {
}
try {
- const supabase = createAdminClient();
+ if (!supabase) {
+ throw new Error("Supabase admin client is unavailable.");
+ }
const [documents, jobs, batches, buckets] = await Promise.all([
supabase.from("documents").select("id,content_hash,import_batch_id").limit(1),
supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts,locked_at").limit(1),
@@ -82,94 +107,177 @@ async function readSchemaStatus() {
}
}
-async function readWorkerStatus() {
+async function readWorkerStatus(supabase: AdminClient | null): Promise {
const label = "npm run worker running";
if (!requiredSupabaseEnvPresent) {
- return check(
- "worker",
- label,
- "unknown",
- "Worker status cannot be inferred until Supabase is configured.",
- );
+ return {
+ check: check("worker", label, "unknown", "Worker status cannot be inferred until Supabase is configured."),
+ activeWork: false,
+ };
}
if (!supabaseProjectCanBeQueried) {
- return check(
- "worker",
- label,
- "unknown",
- "Worker status cannot be inferred while Supabase points at the wrong project.",
- );
+ return {
+ check: check(
+ "worker",
+ label,
+ "unknown",
+ "Worker status cannot be inferred while Supabase points at the wrong project.",
+ ),
+ activeWork: false,
+ };
}
try {
- const supabase = createAdminClient();
+ if (!supabase) {
+ throw new Error("Supabase admin client is unavailable.");
+ }
const [latestResult, activeResult] = await Promise.all([
supabase.from("ingestion_jobs").select("status,updated_at").order("updated_at", { ascending: false }).limit(1),
- supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]),
+ supabase
+ .from("ingestion_jobs")
+ .select("id", { count: "exact", head: true })
+ .in("status", ["pending", "processing"]),
]);
if (latestResult.error || activeResult.error) {
- return check("worker", label, "unknown", "Ingestion jobs could not be checked.");
+ return {
+ check: check("worker", label, "unknown", "Ingestion jobs could not be checked."),
+ activeWork: false,
+ };
}
const latest = latestResult.data?.[0];
+ const activeWork = (activeResult.count ?? 0) > 0;
if (!latest?.updated_at) {
- return check("worker", label, "unknown", "No ingestion activity has been recorded yet.");
+ return {
+ check: check("worker", label, "unknown", "No ingestion activity has been recorded yet."),
+ activeWork,
+ };
}
const updatedAt = new Date(latest.updated_at).getTime();
const recentWindowMs = Math.max(env.WORKER_POLL_MS * 6, 60_000);
if (Number.isFinite(updatedAt) && Date.now() - updatedAt <= recentWindowMs) {
- return check("worker", label, "ready", "Recent ingestion activity was detected.");
+ return {
+ check: check("worker", label, "ready", "Recent ingestion activity was detected."),
+ activeWork,
+ };
}
if ((activeResult.count ?? 0) === 0 && latest.status === "completed") {
- return check("worker", label, "ready", "No queued ingestion work is waiting; the latest job completed.");
+ return {
+ check: check("worker", label, "ready", "No queued ingestion work is waiting; the latest job completed."),
+ activeWork: false,
+ };
}
- return check(
- "worker",
- label,
- "unknown",
- "Queued or processing ingestion work exists, but no recent activity proves the worker is active.",
- );
+ return {
+ check: check(
+ "worker",
+ label,
+ "unknown",
+ "Queued or processing ingestion work exists, but no recent activity proves the worker is active.",
+ ),
+ activeWork,
+ };
} catch {
- return check("worker", label, "unknown", "Worker status could not be inferred.");
+ return {
+ check: check("worker", label, "unknown", "Worker status could not be inferred."),
+ activeWork: false,
+ };
}
}
-export async function GET() {
- const [schema, worker] = await Promise.all([readSchemaStatus(), readWorkerStatus()]);
+function setupStatusCacheTtl(payload: SetupStatusPayload) {
+ return payload.indexingActive ? SETUP_STATUS_ACTIVE_CACHE_MS : SETUP_STATUS_IDLE_CACHE_MS;
+}
+
+function setupStatusResponse(payload: SetupStatusPayload) {
+ return NextResponse.json(payload, {
+ headers: {
+ "Cache-Control": "private, max-age=5, stale-while-revalidate=30",
+ "X-Poll-After-Ms": String(payload.pollAfterMs ?? ""),
+ },
+ });
+}
+
+async function buildSetupStatusPayload(): Promise {
+ const supabase = supabaseProjectCanBeQueried ? createAdminClient() : null;
+ const [schema, worker] = await Promise.all([readSchemaStatus(supabase), readWorkerStatus(supabase)]);
- return NextResponse.json({
+ const checks = [
+ check(
+ "env",
+ ".env.local configured",
+ requiredSupabaseEnvPresent ? "ready" : "needs_setup",
+ requiredSupabaseEnvPresent
+ ? "Required Supabase server environment variables are present."
+ : "Set the required Supabase URL and server key.",
+ ),
+ check(
+ "project",
+ "Clinical KB Database target",
+ supabaseProjectCheck.status === "ready" ? "ready" : "needs_setup",
+ formatSupabaseProjectCheck(supabaseProjectCheck),
+ ),
+ schema,
+ check(
+ "openai",
+ "OpenAI API key available",
+ env.OPENAI_API_KEY ? "ready" : "needs_setup",
+ env.OPENAI_API_KEY
+ ? "OPENAI_API_KEY is present for answers, embeddings, and captions."
+ : "Set OPENAI_API_KEY before real indexing or answers.",
+ ),
+ worker.check,
+ ];
+ const setupSettled = checks.every((item) => item.status === "ready");
+ const pollAfterMs = worker.activeWork ? ACTIVE_INDEXING_POLL_MS : setupSettled ? null : SETUP_RECHECK_POLL_MS;
+
+ return {
demoMode: isDemoMode(),
- checks: [
- check(
- "env",
- ".env.local configured",
- requiredSupabaseEnvPresent ? "ready" : "needs_setup",
- requiredSupabaseEnvPresent
- ? "Required Supabase server environment variables are present."
- : "Set the required Supabase URL and server key.",
- ),
- check(
- "project",
- "Clinical KB Database target",
- supabaseProjectCheck.status === "ready" ? "ready" : "needs_setup",
- formatSupabaseProjectCheck(supabaseProjectCheck),
- ),
- schema,
- check(
- "openai",
- "OpenAI API key available",
- env.OPENAI_API_KEY ? "ready" : "needs_setup",
- env.OPENAI_API_KEY
- ? "OPENAI_API_KEY is present for answers, embeddings, and captions."
- : "Set OPENAI_API_KEY before real indexing or answers.",
- ),
- worker,
- ],
+ checks,
+ indexingActive: worker.activeWork,
+ pollAfterMs,
+ generatedAt: new Date().toISOString(),
+ };
+}
+
+async function readSetupStatusPayload() {
+ const now = Date.now();
+ if (setupStatusCache && setupStatusCache.expiresAt > now) {
+ return setupStatusCache.payload;
+ }
+
+ if (setupStatusInFlight) {
+ return setupStatusInFlight;
+ }
+
+ const promise = buildSetupStatusPayload().then((payload) => {
+ setupStatusCache = {
+ expiresAt: Date.now() + setupStatusCacheTtl(payload),
+ payload,
+ };
+ return payload;
});
+ setupStatusInFlight = promise;
+
+ try {
+ return await promise;
+ } finally {
+ if (setupStatusInFlight === promise) {
+ setupStatusInFlight = null;
+ }
+ }
+}
+
+export async function GET(request: Request) {
+ const identity = localProjectRequestIdentityPayload(request);
+ if (!identity.localServer.safeLocalOrigin) {
+ return unsafeLocalProjectResponse(identity);
+ }
+
+ return setupStatusResponse(await readSetupStatusPayload());
}
diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts
index 9ebbf0b4b..e26238659 100644
--- a/src/app/api/upload/route.ts
+++ b/src/app/api/upload/route.ts
@@ -3,6 +3,7 @@ import { createHash } from "node:crypto";
import { NextResponse } from "next/server";
import { env } from "@/lib/env";
import { assertAllowedFile, jsonError } from "@/lib/http";
+import { planDocumentName, type SupabaseLike } from "@/lib/document-naming";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
@@ -54,7 +55,15 @@ export async function POST(request: Request) {
if (upload.error) throw new Error(upload.error.message);
uploadedPath = storagePath;
- const title = String(formData.get("title") || file.name.replace(/\.[^.]+$/, ""));
+ const namingSupabase = supabase as unknown as SupabaseLike;
+ const namePlan = await planDocumentName({
+ supabase: namingSupabase,
+ ownerId: user.id,
+ fileName: file.name,
+ requestedTitle: formData.get("title") ? String(formData.get("title")) : null,
+ contentHash,
+ });
+ const title = namePlan.title;
const description = formData.get("description") ? String(formData.get("description")) : null;
const uploadedAt = new Date().toISOString();
@@ -81,6 +90,12 @@ export async function POST(request: Request) {
uploaded_at: uploadedAt,
indexed_at: null,
uploaded_by: user.id,
+ original_file_name: namePlan.originalFileName,
+ original_title: namePlan.originalTitle,
+ smart_title_base: namePlan.baseTitle,
+ smart_title_group_key: namePlan.duplicateGroupKey,
+ smart_title_duplicate_index: namePlan.duplicateIndex,
+ smart_title_duplicate_reason: namePlan.duplicateReason,
document_status: "unknown",
clinical_validation_status: "unverified",
extraction_quality: "unknown",
diff --git a/src/app/layout.tsx b/src/app/layout.tsx
index c9e586c6d..78b847a04 100644
--- a/src/app/layout.tsx
+++ b/src/app/layout.tsx
@@ -1,6 +1,5 @@
import type { Metadata, Viewport } from "next";
import { Geist, Geist_Mono } from "next/font/google";
-import Script from "next/script";
import { AuthProvider } from "@/lib/supabase/client";
import "./globals.css";
@@ -30,16 +29,6 @@ export default function RootLayout({
}: Readonly<{
children: React.ReactNode;
}>) {
- const themeScript = `
- (() => {
- try {
- const stored = localStorage.getItem("clinical-kb-theme");
- const dark = stored === "dark" || (!stored && window.matchMedia("(prefers-color-scheme: dark)").matches);
- document.documentElement.classList.toggle("dark", dark);
- } catch {}
- })();
- `;
-
return (
-
{children}
diff --git a/src/components/AccessibleTable.tsx b/src/components/AccessibleTable.tsx
new file mode 100644
index 000000000..b5684e53b
--- /dev/null
+++ b/src/components/AccessibleTable.tsx
@@ -0,0 +1,85 @@
+"use client";
+
+import { cn, textMuted } from "@/components/ui-primitives";
+import { normalizeAccessibleTable } from "@/lib/accessible-table-normalization";
+
+function parseMarkdownTable(markdown?: string | null) {
+ if (!markdown) return null;
+ const rows = markdown
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line.includes("|") && !/^\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?$/.test(line))
+ .map((line) =>
+ line
+ .replace(/^\|/, "")
+ .replace(/\|$/, "")
+ .split("|")
+ .map((cell) => cell.replace(/\\\|/g, "|").trim()),
+ )
+ .filter((row) => row.some(Boolean));
+ return rows.length ? rows : null;
+}
+
+export function AccessibleTable({
+ caption,
+ markdown,
+ rows,
+ columns,
+ compact = false,
+}: {
+ caption?: string | null;
+ markdown?: string | null;
+ rows?: string[][] | null;
+ columns?: string[] | null;
+ compact?: boolean;
+}) {
+ const parsed = rows?.length ? rows : parseMarkdownTable(markdown);
+ if (!parsed?.length) return null;
+
+ const normalized = normalizeAccessibleTable(parsed, columns);
+ if (!normalized) return null;
+
+ const { header, body } = normalized;
+ const visibleBody = body.slice(0, compact ? 6 : 20);
+
+ return (
+
+
+ {caption ? (
+ {caption}
+ ) : null}
+
+
+ {header.map((cell, index) => (
+
+ {cell}
+
+ ))}
+
+
+
+ {visibleBody.map((row, rowIndex) => {
+ return (
+
+ {row.map((cell, cellIndex) => (
+
+ {cell || - }
+
+ ))}
+
+ );
+ })}
+
+
+ {body.length > visibleBody.length ? (
+
+ Showing {visibleBody.length} of {body.length} rows.
+
+ ) : null}
+
+ );
+}
diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx
index a3a6d3204..9fa95d4a3 100644
--- a/src/components/ClinicalDashboard.tsx
+++ b/src/components/ClinicalDashboard.tsx
@@ -32,9 +32,12 @@ import {
X,
} from "lucide-react";
import { FormEvent, ReactNode, useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react";
+import { AccessibleTable } from "@/components/AccessibleTable";
+import { DocumentManagementActions, type DocumentDeleteResult } from "@/components/DocumentManagementActions";
import { documentCitationHref, formatCompactCitationLabel, formatCitationLabel } from "@/lib/citations";
import { extractSafetyFindings, formatSafetyFindingLabel } from "@/lib/clinical-safety";
import { clearCachedSignedUrl, getCachedSignedUrl, setCachedSignedUrl } from "@/lib/signed-url-cache";
+import { isLocalNoAuthMode } from "@/lib/env";
import {
appBackdrop,
answerSurface,
@@ -49,6 +52,7 @@ import {
glassPanel,
iconTilePremium,
fieldLabel,
+ LoadingPanel,
metadataPill,
navPill,
panel,
@@ -68,18 +72,30 @@ import {
toneSuccess,
toneWarning,
} from "@/components/ui-primitives";
-import { useAuthSession } from "@/lib/supabase/client";
+import { AUTH_EMAIL_STORAGE_KEY, useAuthSession } from "@/lib/supabase/client";
import { nextTheme, resolveThemePreference, type ResolvedTheme } from "@/lib/theme";
import { SafeBoldText } from "@/components/SafeBoldText";
+import {
+ parseAnswerDisplayContent,
+ type AnswerDisplayLine,
+ type AnswerDisplayTone,
+ type ParsedAnswerDisplay,
+} from "@/lib/answer-formatting";
+import { sourceTextForDisplay, sourceTextForDisplayPreservingBreaks } from "@/lib/source-text-sanitizer";
+import { smartEvidenceTags } from "@/lib/evidence-tags";
import type {
ClinicalDocument,
BestSourceRecommendation,
+ DocumentMatch,
+ EvidenceRelevance,
ImportBatch,
IngestionJob,
QuoteCard,
RagAnswer,
+ AnswerSection,
RelatedDocument,
SearchResult,
+ SourceEvidenceRelevance,
VisualEvidenceCard,
} from "@/lib/types";
import {
@@ -114,6 +130,10 @@ const navigationHashes = ["#search", "#quotes", "#images", "#sources"] as const;
const themeStorageKey = "clinical-kb-theme";
const themeChangeEvent = "clinical-kb-theme-change";
+const authEmailChangeEvent = "clinical-kb-auth-email-change";
+const documentPageSize = 150;
+const activeIndexingPollFallbackMs = 5_000;
+const setupRecheckPollMs = 60_000;
type SetupCheckStatus = "ready" | "needs_setup" | "unknown";
type SetupCheck = {
@@ -122,8 +142,250 @@ type SetupCheck = {
status: SetupCheckStatus;
detail: string;
};
+type DocumentPagination = {
+ limit: number;
+ offset: number;
+ total: number;
+ nextOffset: number;
+ hasMore: boolean;
+};
+type SearchMode = "answer" | "documents";
+type RefreshOptions = {
+ includeSetup?: boolean;
+ includeDashboardData?: boolean;
+ includeDocumentMeta?: boolean;
+};
+type PollHint = {
+ active?: boolean;
+ pollAfterMs?: number | null;
+};
+type SetupStatusPayload = {
+ demoMode?: boolean;
+ checks?: SetupCheck[];
+ indexingActive?: boolean;
+ pollAfterMs?: number | null;
+};
+type LocalProjectIdentityPayload = {
+ localServer?: {
+ currentUrl?: string | null;
+ projectPortStart?: number;
+ projectPortEnd?: number;
+ safeLocalOrigin?: boolean;
+ };
+};
+type DocumentsPayload = {
+ documents?: ClinicalDocument[];
+ pagination?: DocumentPagination | null;
+ demoMode?: boolean;
+ setupRequired?: boolean;
+ error?: string;
+ indexing?: PollHint;
+};
+type JobsPayload = {
+ jobs?: IngestionJob[];
+ demoMode?: boolean;
+ setupRequired?: boolean;
+ error?: string;
+ hasActiveJobs?: boolean;
+ pollAfterMs?: number | null;
+};
+type BatchesPayload = {
+ batches?: ImportBatch[];
+ demoMode?: boolean;
+ hasActiveBatches?: boolean;
+ pollAfterMs?: number | null;
+};
type AnswerPayload = RagAnswer & { demoMode?: boolean };
+type SearchResultModePayload =
+ | {
+ kind: "documents";
+ query: string;
+ demoMode?: boolean;
+ sources: SearchResult[];
+ documentMatches: DocumentMatch[];
+ relevance?: EvidenceRelevance;
+ }
+ | {
+ kind: "answer";
+ query: string;
+ payload: AnswerPayload;
+ };
+
+type SearchError = Error & {
+ status?: number;
+ retryable?: boolean;
+};
+
+const searchRetryDelaysMs = [500, 1000, 2000] as const;
+const searchRetryCount = 2;
+const keywordStopWords = new Set([
+ "a",
+ "about",
+ "all",
+ "an",
+ "and",
+ "are",
+ "as",
+ "at",
+ "be",
+ "before",
+ "both",
+ "by",
+ "can",
+ "could",
+ "did",
+ "do",
+ "does",
+ "for",
+ "from",
+ "get",
+ "had",
+ "has",
+ "have",
+ "her",
+ "his",
+ "how",
+ "if",
+ "in",
+ "is",
+ "it",
+ "its",
+ "into",
+ "me",
+ "may",
+ "more",
+ "my",
+ "no",
+ "not",
+ "of",
+ "on",
+ "or",
+ "our",
+ "out",
+ "should",
+ "so",
+ "such",
+ "that",
+ "the",
+ "their",
+ "them",
+ "there",
+ "these",
+ "they",
+ "this",
+ "those",
+ "to",
+ "when",
+ "where",
+ "which",
+ "who",
+ "why",
+ "with",
+ "would",
+ "you",
+]);
+
+function makeSearchError(message: string, status?: number, retryable = false): SearchError {
+ const error = new Error(message) as SearchError;
+ error.status = status;
+ error.retryable = retryable;
+ return error;
+}
+
+function isRetryableStatus(status: number) {
+ return status === 408 || status === 429 || (status >= 500 && status <= 599);
+}
+
+function isRetryableMessage(message: string) {
+ const normalized = message.toLowerCase();
+ return (
+ normalized.includes("failed to fetch") ||
+ normalized.includes("network") ||
+ normalized.includes("timeout") ||
+ normalized.includes("timed out") ||
+ normalized.includes("rate limit") ||
+ normalized.includes("rate-limited") ||
+ normalized.includes("temporar") ||
+ normalized.includes("overload") ||
+ normalized.includes("retry") ||
+ normalized.includes("unavailable") ||
+ normalized.includes("upstream") ||
+ normalized.includes("service is currently")
+ );
+}
+
+function isRetryableError(error: unknown) {
+ if (!(error instanceof Error)) return false;
+
+ const searchError = error as SearchError;
+ if (searchError.name === "TypeError") return true;
+ if (searchError.retryable !== undefined) return searchError.retryable;
+ if (searchError.status !== undefined) return isRetryableStatus(searchError.status);
+ return isRetryableMessage(searchError.message);
+}
+
+function sleep(ms: number) {
+ return new Promise((resolve) => window.setTimeout(resolve, ms));
+}
+
+function keywordQueryFromNaturalLanguage(query: string) {
+ const normalized = query
+ .normalize("NFKD")
+ .toLowerCase()
+ .replace(/[^\w\s]+/g, " ")
+ .replace(/_/g, " ")
+ .trim();
+ const tokens = normalized.split(/\s+/).filter((token) => token.length >= 3 && !keywordStopWords.has(token));
+ const terms: string[] = [];
+ const seen = new Set();
+
+ for (const token of tokens) {
+ if (seen.has(token)) continue;
+ seen.add(token);
+ terms.push(token);
+ }
+
+ return terms.slice(0, 7).join(" ");
+}
+
+function answerPayloadIsUsable(payload: AnswerPayload) {
+ const answerText = payload.answer.trim();
+ if (!answerText) return false;
+ if (payload.confidence === "unsupported") return false;
+
+ return true;
+}
+
+function progressForRetry(attempt: number) {
+ if (attempt <= 1) return "Retrying...";
+ return `Retrying... (${Math.min(attempt, searchRetryCount)}/${searchRetryCount})`;
+}
+
+async function readLocalProjectIdentity() {
+ const response = await fetch("/api/local-project-id", { cache: "no-store" });
+ if (!response.ok) return null;
+ return (await response.json()) as LocalProjectIdentityPayload;
+}
+
+function unsafeLocalProjectMessage(identity: LocalProjectIdentityPayload | null) {
+ const range =
+ typeof identity?.localServer?.projectPortStart === "number" &&
+ typeof identity.localServer.projectPortEnd === "number"
+ ? ` Use the URL printed by npm run ensure; managed ports are ${identity.localServer.projectPortStart}-${identity.localServer.projectPortEnd}.`
+ : " Use the URL printed by npm run ensure.";
+ return `This tab is not using the guarded Clinical KB local URL.${range}`;
+}
+
+function parseSseData(lines: string[]) {
+ const data = lines.join("\n").trim();
+ if (!data) return null;
+ try {
+ return JSON.parse(data);
+ } catch {
+ throw makeSearchError("Answer stream returned malformed data.", 500, true);
+ }
+}
function answerStreamProgressMessage(data: unknown) {
if (!data || typeof data !== "object") return null;
@@ -132,7 +394,7 @@ function answerStreamProgressMessage(data: unknown) {
}
async function readAnswerStream(response: Response, onProgress: (message: string) => void): Promise {
- if (!response.body) throw new Error("Answer stream could not be opened.");
+ if (!response.body) throw makeSearchError("Answer stream could not be opened.", undefined, true);
const reader = response.body.getReader();
const decoder = new TextDecoder();
@@ -150,7 +412,8 @@ async function readAnswerStream(response: Response, onProgress: (message: string
}
if (dataLines.length === 0) return;
- const data = JSON.parse(dataLines.join("\n")) as unknown;
+ const data = parseSseData(dataLines);
+ if (data === null) return;
if (event === "progress") {
const message = answerStreamProgressMessage(data);
if (message) onProgress(message);
@@ -158,7 +421,25 @@ async function readAnswerStream(response: Response, onProgress: (message: string
}
if (event === "error") {
const message = data && typeof data === "object" ? (data as { error?: unknown }).error : null;
- throw new Error(typeof message === "string" && message ? message : "Answer generation failed.");
+ const details =
+ data && typeof data === "object" ? (data as { details?: { message?: unknown } | unknown }).details : null;
+ const detailMessage =
+ details && typeof details === "object" && "message" in details && typeof details.message === "string"
+ ? details.message
+ : null;
+ const status = data && typeof data === "object" ? (data as { status?: unknown }).status : null;
+ const statusCode = typeof status === "number" ? status : undefined;
+ const errorMessage =
+ typeof message === "string" && message.trim()
+ ? message
+ : typeof detailMessage === "string" && detailMessage.trim()
+ ? detailMessage
+ : "Answer generation failed due to a streaming error.";
+ throw makeSearchError(
+ errorMessage,
+ statusCode,
+ isRetryableStatus(statusCode ?? 0) || isRetryableMessage(errorMessage),
+ );
}
if (event === "final") {
finalPayload = data as AnswerPayload;
@@ -181,7 +462,7 @@ async function readAnswerStream(response: Response, onProgress: (message: string
}
if (buffer.trim()) processEvent(buffer.trim());
- if (!finalPayload) throw new Error("Answer stream ended before a final answer was received.");
+ if (!finalPayload) throw makeSearchError("Answer stream ended before a final answer was received.", undefined, true);
return finalPayload as AnswerPayload;
}
@@ -216,6 +497,32 @@ function subscribeTheme(onStoreChange: () => void) {
};
}
+function getAuthEmailSnapshot() {
+ if (typeof window === "undefined") return "";
+ try {
+ return window.localStorage.getItem(AUTH_EMAIL_STORAGE_KEY) ?? "";
+ } catch {
+ return "";
+ }
+}
+
+function getServerAuthEmailSnapshot() {
+ return "";
+}
+
+function subscribeAuthEmail(onStoreChange: () => void) {
+ if (typeof window === "undefined") return () => undefined;
+ const notify = () => onStoreChange();
+
+ window.addEventListener("storage", notify);
+ window.addEventListener(authEmailChangeEvent, notify);
+
+ return () => {
+ window.removeEventListener("storage", notify);
+ window.removeEventListener(authEmailChangeEvent, notify);
+ };
+}
+
function useTheme() {
const theme = useSyncExternalStore(subscribeTheme, getThemeSnapshot, getServerThemeSnapshot);
@@ -439,13 +746,98 @@ function SectionHeading({
);
}
+function relevanceChipLabel(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, grounded = false) {
+ if (!relevance) return grounded ? "Source-backed" : "No direct support";
+ if (relevance.verdict === "direct") return "Source-backed";
+ if (relevance.verdict === "partial") return "Partial support";
+ if (relevance.verdict === "nearby") return "Nearby only";
+ return "No direct support";
+}
+
+function relevanceChipClasses(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, grounded = false) {
+ const verdict = relevance?.verdict ?? (grounded ? "direct" : "none");
+ if (verdict === "direct") {
+ return "border-[color:var(--success)]/20 bg-[color:var(--success-soft)]/45 text-[color:var(--success)]";
+ }
+ if (verdict === "partial") {
+ return "border-[color:var(--primary)]/25 bg-[color:var(--primary-soft)]/45 text-[color:var(--primary)]";
+ }
+ return "border-[color:var(--warning)]/25 bg-[color:var(--warning-soft)]/45 text-[color:var(--warning)]";
+}
+
+function relevanceIcon(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined, grounded = false) {
+ const verdict = relevance?.verdict ?? (grounded ? "direct" : "none");
+ return verdict === "direct" || verdict === "partial" ? CheckCircle2 : AlertCircle;
+}
+
+function isWeakRelevance(relevance: EvidenceRelevance | SourceEvidenceRelevance | null | undefined) {
+ return !relevance?.isSourceBacked || relevance.verdict === "nearby" || relevance.verdict === "none";
+}
+
+function RelevanceBadge({
+ relevance,
+ grounded = false,
+ testId,
+}: {
+ relevance?: EvidenceRelevance | SourceEvidenceRelevance | null;
+ grounded?: boolean;
+ testId?: string;
+}) {
+ const Icon = relevanceIcon(relevance, grounded);
+ const label = relevanceChipLabel(relevance, grounded);
+ return (
+
+
+ {label}
+
+ );
+}
+
+function QueryCoverageChips({
+ relevance,
+ limit = 4,
+}: {
+ relevance?: SourceEvidenceRelevance | EvidenceRelevance | null;
+ limit?: number;
+}) {
+ if (!relevance) return null;
+ const chips =
+ "chips" in relevance && relevance.chips.length
+ ? relevance.chips
+ : [
+ relevance.matchedTerms.length ? `matched: ${relevance.matchedTerms.slice(0, 3).join(", ")}` : "",
+ relevance.missingTerms.length ? `missing: ${relevance.missingTerms.slice(0, 3).join(", ")}` : "",
+ relevanceChipLabel(relevance),
+ ].filter(Boolean);
+ return (
+
+ {chips.slice(0, limit).map((chip) => (
+
+ {chip}
+
+ ))}
+
+ );
+}
+
function AnswerHeaderActions({
bestSource,
grounded,
+ relevance,
}: {
bestSource: BestSourceRecommendation | null;
grounded: boolean;
+ relevance?: EvidenceRelevance | null;
}) {
+ const sourceLabel = relevance && !relevance.isSourceBacked ? "Closest source" : "Top source";
return (
-
Top
-
Top source
+
{sourceLabel === "Closest source" ? "Closest" : "Top"}
+
{sourceLabel}
)}
- {grounded ? (
-
-
- Backed
- Source-backed
-
- ) : (
-
-
- Limited
- Insufficient
-
- )}
+
);
}
@@ -491,6 +863,7 @@ function AnswerHeaderActions({
function MasterSearchHeader({
documents,
query,
+ searchMode,
loading,
selectedDocumentIds,
hasAnswer,
@@ -498,6 +871,7 @@ function MasterSearchHeader({
realDataReady,
theme,
onQueryChange,
+ onSearchModeChange,
onAsk,
onClearQuery,
onClearScope,
@@ -507,6 +881,7 @@ function MasterSearchHeader({
}: {
documents: ClinicalDocument[];
query: string;
+ searchMode: SearchMode;
loading: boolean;
selectedDocumentIds: string[];
hasAnswer: boolean;
@@ -514,6 +889,7 @@ function MasterSearchHeader({
realDataReady: boolean;
theme: ResolvedTheme;
onQueryChange: (query: string) => void;
+ onSearchModeChange: (mode: SearchMode) => void;
onAsk: () => void;
onClearQuery: () => void;
onClearScope: () => void;
@@ -521,8 +897,18 @@ function MasterSearchHeader({
onPickSample: (sample: string) => void;
onToggleTheme: () => void;
}) {
- const canAsk = query.trim().length >= 2 && !loading && realDataReady;
+ const trimmedQuery = query.trim();
+ const canAsk = trimmedQuery.length >= 1 && !loading && realDataReady;
const compactMobile = hasAnswer;
+ const selectedDocuments = selectedDocumentIds.map((id) => documents.find((document) => document.id === id)).filter(Boolean);
+ const scopeSummary = selectedDocumentIds.length === 0 ? "All documents" : `${selectedDocumentIds.length} scoped`;
+ const scopePreview = selectedDocuments
+ .slice(0, 2)
+ .map((document) => document?.title.replace(/^Synthetic /, ""))
+ .filter(Boolean)
+ .join(", ");
+ const submitShortLabel = searchMode === "answer" ? "Ask" : "Docs";
+ const submitFullLabel = searchMode === "answer" ? (trimmedQuery ? "Answer" : "Ask") : "Find docs";
function submit(event: FormEvent) {
event.preventDefault();
@@ -531,47 +917,57 @@ function MasterSearchHeader({
function renderScopeAndPromptRows() {
return (
-
-
-
-
- All documents
-
- {documents.map((document) => {
- const selected = selectedDocumentIds.includes(document.id);
- return (
-
onToggleScope(document.id)}
- title={document.title}
- className={cn(
- shellChip,
- "max-w-[13rem] sm:max-w-[15rem]",
- selected
- ? "border-teal-300/40 bg-teal-300/18 text-teal-50"
- : "border-white/12 bg-white/6 text-slate-200 hover:bg-white/10",
- )}
- >
-
- {document.title.replace(/^Synthetic /, "")}
-
- );
- })}
+
+
+
+
Document scope
+
{documents.length} available
-
+
+
+
+ All documents
+
+ {documents.map((document) => {
+ const selected = selectedDocumentIds.includes(document.id);
+ return (
+ onToggleScope(document.id)}
+ title={document.title}
+ className={cn(
+ shellChip,
+ "max-w-full sm:max-w-[15rem]",
+ selected
+ ? "border-teal-300/40 bg-teal-300/18 text-teal-50"
+ : "border-white/12 bg-white/6 text-slate-200 hover:bg-white/10",
+ )}
+ >
+
+ {document.title.replace(/^Synthetic /, "")}
+
+ );
+ })}
+
+
+
-
-
+
+
+
Sample prompts
+
{sampleQueries.length} prompts
+
+
{sampleQueries.map((sample) => (
))}
-
+
);
}
@@ -645,6 +1041,41 @@ function MasterSearchHeader({
+
+
+ {[
+ { mode: "answer" as const, label: "Answer", icon: Sparkles },
+ { mode: "documents" as const, label: "Documents", icon: FileText },
+ ].map((item) => {
+ const active = searchMode === item.mode;
+ const Icon = item.icon;
+ return (
+ onSearchModeChange(item.mode)}
+ className={cn(
+ "inline-flex h-9 items-center justify-center gap-2 rounded-md px-3 text-xs font-semibold transition",
+ active
+ ? "bg-white text-slate-950 shadow-[var(--shadow-tight)]"
+ : "text-slate-200 hover:bg-white/10 hover:text-white",
+ )}
+ aria-pressed={active}
+ aria-label={
+ item.mode === "answer" ? "Switch to answer mode" : "Switch to document search mode"
+ }
+ >
+
+ {item.label}
+
+ );
+ })}
+
+
+ {searchMode === "answer" ? "Synthesize cited clinical guidance" : "List matching source documents"}
+
+
+
- {!hasAnswer ? (
- {renderScopeAndPromptRows()}
- ) : (
-
-
-
-
- Scope & prompts
- {selectedDocumentIds.length > 0 && (
-
- {selectedDocumentIds.length} scoped
-
- )}
-
+
+
+
+
+ Scope & prompts
+ {scopeSummary}
+ {scopePreview ? {scopePreview} : null}
+
+
+ {documents.length} documents
-
- {renderScopeAndPromptRows()}
-
- )}
+
+
+ {renderScopeAndPromptRows()}
+
);
@@ -869,6 +1305,182 @@ function VerificationActionStrip({
);
}
+function sourceResultHref(source: SearchResult) {
+ return `/documents/${source.document_id}?page=${source.page_number ?? 1}&chunk=${source.id}`;
+}
+
+function SourcePassageLinks({
+ heading,
+ sources,
+ compact = false,
+}: {
+ heading: string;
+ sources: SearchResult[];
+ compact?: boolean;
+}) {
+ if (sources.length === 0) return null;
+
+ return (
+
+ {sources.slice(0, compact ? 2 : 3).map((source, index) => (
+
+
+ p.{source.page_number ?? "n/a"}
+ chunk {source.chunk_index}
+ {source.source_strength ? · {source.source_strength} : null}
+
+ ))}
+
+ );
+}
+
+function SourceLinkedAnswer({
+ sections,
+ fallbackText,
+}: {
+ sections: Array;
+ fallbackText: string;
+}) {
+ const demoNotice = fallbackText.match(/Synthetic demo only:.*$/i)?.[0] ?? null;
+
+ if (sections.length === 0) {
+ return (
+
+ );
+ }
+
+ return (
+
+ {sections.map((section, index) => {
+ const sectionContent = sectionBodyContent(section.body);
+ return (
+
+
+
+
{index + 1}
+
+
+ {section.heading}
+
+
+ {section.citationSources.length} source passage
+ {section.citationSources.length === 1 ? "" : "s"}
+
+
+
+
+
+
+
+
+ );
+ })}
+ {demoNotice ? (
+
+ {demoNotice}
+
+ ) : null}
+
+ );
+}
+
+function answerToneClasses(tone: AnswerDisplayTone) {
+ if (tone === "risk" || tone === "gap") return toneDanger;
+ if (tone === "monitoring" || tone === "medication") return toneWarning;
+ if (tone === "action" || tone === "direct") return toneSuccess;
+ if (tone === "comparison" || tone === "source") return toneInfo;
+ return toneNeutral;
+}
+
+function AnswerSymbolTile({ line }: { line: AnswerDisplayLine }) {
+ return (
+
+ {line.presentation.symbol}
+
+ );
+}
+
+function AnswerLineLabel({ line }: { line: AnswerDisplayLine }) {
+ const label = line.label ?? line.presentation.label;
+ return (
+
+ {label}
+
+ );
+}
+
+function FormattedAnswerContent({ content }: { content: ParsedAnswerDisplay }) {
+ if (content.type === "paragraph") {
+ const line = content.lines[0];
+ return (
+
+ {line ?
: null}
+
+ {line ? : null}
+ {line ? " " : null}
+
+
+
+ );
+ }
+
+ const listClasses =
+ content.mode === "clinical_pathway" || content.mode === "evidence_gap"
+ ? "space-y-2.5"
+ : content.mode === "comparison"
+ ? "grid gap-2.5 md:grid-cols-2"
+ : "space-y-2.5";
+
+ return (
+
+ {content.lines.map((line) => (
+
+
+
+
+
+
+ ))}
+
+ );
+}
+
function BestSourceCard({
source,
grounded,
@@ -901,12 +1513,16 @@ function BestSourceCard({
+
{score}% match
+
+
+
Reason selected
“{source.snippet}”
@@ -942,8 +1558,11 @@ function SafetyFindingsPanel({ findings }: { findings: ReturnType
- {findings.map((finding) => (
-
+ {findings.map((finding, index) => (
+
{finding.label}
@@ -971,14 +1590,93 @@ function SafetyFindingsPanel({ findings }: { findings: ReturnType
+ }
+ />
+
+
+ What was found
+ {found}
+
+
+ What was not found
+ {missing}
+
+
+ Closest sources
+ {closestSources.length ? (
+
+ {closestSources.map((source) => (
+
+
+ {source.title}
+
+ ))}
+
+ ) : (
+ No nearby indexed sources were returned.
+ )}
+
+
+
+ Suggested next search/upload
+
+
+ Try a narrower query using the missing terms, scope to a likely document, or upload/index the guideline that
+ directly covers "{query.trim()}".
+
+
+
+
+ );
+}
+
function CopyGovernanceStrip({
copiedAnswer,
copiedDraft,
+ weakEvidence = false,
onCopyAnswer,
onCopyDraft,
}: {
copiedAnswer: boolean;
copiedDraft: boolean;
+ weakEvidence?: boolean;
onCopyAnswer: () => void;
onCopyDraft: () => void;
}) {
@@ -991,7 +1689,9 @@ function CopyGovernanceStrip({
)}
>
- Draft only; verify source first before pasting into the medical record.
+ {weakEvidence
+ ? "Weak source support; copy only as a search note, not clinical guidance."
+ : "Draft only; verify source first before pasting into the medical record."}
@@ -1078,89 +1778,142 @@ function QuoteCards({
function ClinicalOutputPanel({
answer,
+ demoMode,
copiedWardNote,
onCopyWardNote,
+ collapsed = false,
}: {
answer: RagAnswer;
+ demoMode: boolean;
copiedWardNote: boolean;
onCopyWardNote: () => void;
+ collapsed?: boolean;
}) {
const sections = buildClinicalOutputSections(answer);
if (sections.length === 0) return null;
- return (
- <>
-
-
-
-
-
-
-
- Clinical formats
- {sections.length} practical formats
-
-
-
-
-
-
-
- Draft only; verify source first before pasting into the medical record.
-
- {sections.map((section) => (
-
- {section.title}
-
- {section.items.map((item) => (
-
-
- {item}
-
- ))}
-
-
- ))}
-
-
-
-
- }
- />
-
- Draft only; verify source first before pasting into the medical record.
+ const content = (
+
+ }
+ hideDescriptionOnMobile
+ compactMobile
+ />
+
+ Draft only; verify source first before pasting into the medical record.
+
+ {demoMode ? (
+
+ Synthetic demo only: this is not clinical guidance.
-
- {sections.map((section) => (
-
- {section.title}
-
+ ))}
+
+
);
+
+ if (collapsed) {
+ return (
+
+ {content}
+
+ );
+ }
+
+ return content;
}
-function VisualEvidenceStrip({ evidence }: { evidence: VisualEvidenceCard[] }) {
- return (
-
+function VisualEvidenceStrip({
+ evidence,
+ collapsed = false,
+ embedded = false,
+}: {
+ evidence: VisualEvidenceCard[];
+ collapsed?: boolean;
+ embedded?: boolean;
+}) {
+ function looksLikeTableText(value?: string | null) {
+ return Boolean(value?.includes("|") && value.split("|").filter((cell) => cell.trim()).length >= 3);
+ }
+
+ if (collapsed) {
+ return (
+
+ );
+ }
+
+ const content = (
+ <>
@@ -1172,40 +1925,85 @@ function VisualEvidenceStrip({ evidence }: { evidence: VisualEvidenceCard[] }) {
/>
) : (
- {evidence.map((item) => (
-
-
-
-
-
- {item.caption}
-
-
-
- {formatCompactCitationLabel(item)}
-
-
- {item.title}, page {item.page_number ?? "n/a"}
-
- {item.image_type && (
-
- {item.image_type.replaceAll("_", " ")}
+ {evidence.map((item) => {
+ const tableMarkdown = item.accessibleTableMarkdown?.trim()
+ ? item.accessibleTableMarkdown
+ : looksLikeTableText(item.tableTextSnippet)
+ ? item.tableTextSnippet
+ : null;
+ const hasStructuredTable = Boolean(tableMarkdown || item.tableRows?.length || item.tableColumns?.length);
+ const displayLabels = smartEvidenceTags(
+ item.labels,
+ [[item.tableLabel, item.tableTitle].filter(Boolean).join(": "), item.caption, item.tableTextSnippet]
+ .filter(Boolean)
+ .join(" "),
+ );
+ return (
+
+
+
+
+
+ {[item.tableLabel, item.tableTitle].filter(Boolean).length > 0 && (
+ {[item.tableLabel, item.tableTitle].filter(Boolean).join(": ")}
+ )}
+ {item.caption}
+
+ {!hasStructuredTable && item.tableTextSnippet ? (
+ {item.tableTextSnippet}
+ ) : null}
+ {displayLabels.length ? (
+
+ {displayLabels.map((label) => (
+
+ {label}
+
+ ))}
+
+ ) : null}
+
+
+
+ {formatCompactCitationLabel(item)}
- )}
-
-
- Open PDF
-
-
-
- ))}
+
+ {item.title}, page {item.page_number ?? "n/a"}
+
+ {item.image_type && (
+
+ {item.image_type.replaceAll("_", " ")}
+
+ )}
+
+
+
+ Open PDF
+
+
+
+ );
+ })}
)}
+ >
+ );
+
+ if (embedded) return {content}
;
+
+ return (
+
);
}
@@ -1229,46 +2027,280 @@ function RelatedDocumentsPanel({
{documents.map((document) => (
-
+
+
+
+
{document.title}
+
+
+ {document.match_reason} · pages {document.best_pages.join(", ") || "n/a"} · {document.image_count}{" "}
+ images{document.table_count ? ` · ${document.table_count} tables` : ""}
+
+
+
onScopeDocument(document.document_id)}
+ className={cn(floatingControl, "min-h-[44px] px-3 text-xs")}
+ >
+ Scope
+
+
+ {document.summary && (
+
+
+
+ )}
+ {document.labels.length > 0 && (
+
+ {document.labels.slice(0, 6).map((label) => (
+
+ {label.label}
+
+ ))}
+
+ )}
+
+ ))}
+
+
+ );
+}
+
+function DocumentSearchResultsPanel({
+ matches,
+ query,
+ loading,
+ documentCount,
+ realDataReady,
+ authUnavailable,
+ apiUnavailable,
+ setupWarning,
+ relevance,
+ onScopeDocument,
+ onAnswerFromDocument,
+}: {
+ matches: DocumentMatch[];
+ query: string;
+ loading: boolean;
+ documentCount: number;
+ realDataReady: boolean;
+ authUnavailable: boolean;
+ apiUnavailable: boolean;
+ setupWarning: string | null;
+ relevance?: EvidenceRelevance | null;
+ onScopeDocument: (documentId: string) => void;
+ onAnswerFromDocument: (documentId: string) => void;
+}) {
+ const trimmedQuery = query.trim();
+
+ if (loading) return ;
+
+ if (apiUnavailable || !realDataReady || authUnavailable) {
+ return (
+
+ );
+ }
+
+ if (matches.length === 0) {
+ if (documentCount === 0) {
+ return (
+
+ );
+ }
+
+ if (!trimmedQuery) {
+ return (
+
+ );
+ }
+
+ return (
+
+ );
+ }
+
+ return (
+
+
+
+ {matches.length} document match{matches.length === 1 ? "" : "es"} for "{query.trim()}"
+
+ {relevance ?
: null}
+
+
+ {matches.map((document) => (
+
+
{document.title}
-
- {document.match_reason} · pages {document.best_pages.join(", ") || "n/a"} ·{" "}
- {document.image_count} images
+
+ {document.file_name} · pages {document.bestPages.join(", ") || "n/a"} · {document.tableCount} tables ·{" "}
+ {document.imageCount} images
+
{document.matchReason}
+
+
+
+
+
+
+
+
+ Open
+
+ onScopeDocument(document.document_id)}
+ className={cn(floatingControl, "min-h-[44px] px-3 text-xs")}
+ aria-label={`Scope search to ${document.title}`}
+ >
+
+ Scope
+
+ onAnswerFromDocument(document.document_id)}
+ className={cn(primaryControl, "min-h-[44px] rounded-lg px-3 text-xs")}
+ aria-label={`Answer from ${document.title}`}
+ >
+
+ Answer
+
-
onScopeDocument(document.document_id)}
- className={cn(floatingControl, "min-h-[44px] px-3 text-xs")}
- >
- Scope
-
- {document.summary && (
-
-
+ {document.summarySnippet && (
+
+
)}
{document.labels.length > 0 && (
- {document.labels.slice(0, 6).map((label) => (
-
+ {document.labels.slice(0, 4).map((label) => (
+
{label.label}
))}
+ {document.labels.length > 4 ? (
+
+ +{document.labels.length - 4} more
+
+ ) : null}
)}
))}
-
+
+ );
+}
+
+const displayJsonArtifactPattern =
+ /"?(answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)"?\s*:\s*/i;
+
+function normalizeDisplayText(value: string) {
+ return value.trim().replace(/\s+/g, " ");
+}
+
+type DisplayTextSanitizeOptions = {
+ minLength?: number;
+ minTokens?: number;
+};
+
+function looksLikeDisplayArtifact(value: string) {
+ const normalized = normalizeDisplayText(value);
+ if (!normalized) return true;
+ const quoteCount = (normalized.match(/"/g) ?? []).length;
+ const colonCount = (normalized.match(/:/g) ?? []).length;
+ if (normalized.startsWith("{") && normalized.endsWith("}") && displayJsonArtifactPattern.test(normalized))
+ return true;
+ if (/[{}\[\]]/.test(normalized) && quoteCount >= 4 && colonCount >= 2 && displayJsonArtifactPattern.test(normalized))
+ return true;
+ return false;
+}
+
+function sanitizeDisplayText(value: string, options: DisplayTextSanitizeOptions = {}) {
+ const normalized = normalizeDisplayText(sourceTextForDisplay(value));
+ if (!normalized) return "";
+ const artifactStart = normalized.search(
+ /\{\s*"(?:answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|source_chunk_ids|chunk_id|conflictsOrGaps|quoteCards?)\s*:/i,
+ );
+ const trimmed =
+ artifactStart === -1 ? normalized : artifactStart === 0 ? "" : normalized.slice(0, artifactStart).trim();
+ if (!trimmed) return "";
+ const { minLength = 2, minTokens = 1 } = options;
+ if (trimmed.length < minLength) return "";
+ const tokenCount = trimmed.split(/\s+/).filter(Boolean).length;
+ if (tokenCount < minTokens) return "";
+ if (!/[A-Za-z]{2,}/.test(trimmed)) return "";
+ return looksLikeDisplayArtifact(trimmed) ? "" : trimmed;
+}
+
+function sanitizeAnswerDisplayText(value: string, options: DisplayTextSanitizeOptions = {}) {
+ const normalized = sourceTextForDisplayPreservingBreaks(value).trim();
+ if (!normalized) return "";
+ const artifactStart = normalizeDisplayText(normalized).search(
+ /\{\s*"(?:answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|source_chunk_ids|chunk_id|conflictsOrGaps|quoteCards?)\s*:/i,
);
+ const trimmed =
+ artifactStart === -1 ? normalized : artifactStart === 0 ? "" : normalized.slice(0, artifactStart).trim();
+ if (!trimmed) return "";
+ const { minLength = 2, minTokens = 1 } = options;
+ if (trimmed.length < minLength) return "";
+ const tokenCount = normalizeDisplayText(trimmed).split(/\s+/).filter(Boolean).length;
+ if (tokenCount < minTokens) return "";
+ if (!/[A-Za-z]{2,}/.test(trimmed)) return "";
+ return looksLikeDisplayArtifact(trimmed) ? "" : trimmed;
+}
+
+function sectionBodyContent(body: string) {
+ const normalized = sanitizeAnswerDisplayText(body, { minLength: 8, minTokens: 2 });
+ if (!normalized) {
+ return {
+ ...parseAnswerDisplayContent("No usable section text available."),
+ safe: false,
+ };
+ }
+ return { ...parseAnswerDisplayContent(normalized), safe: true };
}
function SourceList({
@@ -1288,35 +2320,51 @@ function SourceList({
{sources.map((source) => (
-
-
-
- {source.title}
-
-
- {source.file_name} · page {source.page_number ?? "n/a"} · chunk {source.chunk_index}
-
-
-
-
-
-
- onScopeDocument(source.document_id)}
- className="inline-flex min-h-[44px] items-center gap-2 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] px-3 text-xs font-semibold text-[color:var(--text-muted)] transition hover:bg-[color:var(--surface-subtle)]"
- >
-
- This document
-
-
-
-
- {source.content}
-
+ {(() => {
+ const snippet = sanitizeDisplayText(source.content);
+ const fallback = "No usable snippet text for this passage.";
+
+ return (
+ <>
+
+
+
+ {source.title}
+
+
+ {source.file_name} · page {source.page_number ?? "n/a"} · chunk {source.chunk_index}
+
+
+
+
+
+
+
+
+
+
+ Page {source.page_number ?? "n/a"}
+ Chunk {source.chunk_index}
+ onScopeDocument(source.document_id)}
+ className="inline-flex min-h-[44px] items-center gap-2 rounded-lg border border-[color:var(--border)] bg-[color:var(--surface)] px-3 text-xs font-semibold text-[color:var(--text-muted)] transition hover:bg-[color:var(--surface-subtle)]"
+ aria-label={`Scope search to ${source.title}`}
+ >
+
+ This document
+
+
+
+
+ {snippet ? : {fallback} }
+
+ >
+ );
+ })()}
))}
@@ -1352,8 +2400,11 @@ function AnswerSkeleton() {
function AuthPanel() {
const { status, error, isConfigured, signInWithEmail, signOut, session } = useAuthSession();
- const [email, setEmail] = useState("");
+ const savedEmail = useSyncExternalStore(subscribeAuthEmail, getAuthEmailSnapshot, getServerAuthEmailSnapshot);
+ const [draftEmail, setDraftEmail] = useState(null);
+ const email = draftEmail ?? savedEmail;
const busy = status === "loading";
+ const isExpired = status === "expired";
async function submit(event: FormEvent) {
event.preventDefault();
@@ -1400,10 +2451,12 @@ function AuthPanel() {
- {status === "expired" ? "Session expired" : "Sign in for private documents"}
+ {isExpired ? "Sign-in link expired" : "Sign in for private documents"}
- Real-data search, upload, and source previews require a Supabase Auth session.
+ {isExpired
+ ? "Send a fresh link if this one failed or already timed out."
+ : "Real-data search, upload, and source previews require a Supabase Auth session."}
@@ -1414,7 +2467,7 @@ function AuthPanel() {
setEmail(event.target.value)}
+ onChange={(event) => setDraftEmail(event.target.value)}
placeholder="you@example.com"
className={fieldControlWithIcon}
/>
@@ -1440,12 +2493,24 @@ function AuthPanel() {
function DocumentDrawer({
documents,
+ pagination,
+ loadingMoreDocuments,
selectedDocumentIds,
onToggleScope,
+ onLoadMoreDocuments,
+ onDocumentRenamed,
+ onDocumentDeleted,
+ canManageDocuments,
}: {
documents: ClinicalDocument[];
+ pagination: DocumentPagination | null;
+ loadingMoreDocuments: boolean;
selectedDocumentIds: string[];
onToggleScope: (documentId: string) => void;
+ onLoadMoreDocuments: () => void;
+ onDocumentRenamed: (document: ClinicalDocument) => void;
+ onDocumentDeleted: (result: DocumentDeleteResult) => void;
+ canManageDocuments: boolean;
}) {
const [filter, setFilter] = useState("");
const filtered = documents.filter((document) => {
@@ -1466,6 +2531,11 @@ function DocumentDrawer({
className={fieldControlWithIcon}
/>
+ {pagination && pagination.total > documents.length ? (
+
+ Showing {documents.length} of {pagination.total} documents. Load more to manage older files.
+
+ ) : null}
{filtered.length === 0 ? (
+
onToggleScope(document.id)}
@@ -1533,6 +2609,17 @@ function DocumentDrawer({
})
)}
+ {pagination?.hasMore ? (
+
+ {loadingMoreDocuments ? : }
+ Load more documents
+
+ ) : null}
);
}
@@ -1671,7 +2758,9 @@ function IndexingMonitor({
onEnrich: (documentId: string) => void;
}) {
if (jobs.length === 0 && batches.length === 0) {
- return ;
+ return (
+
+ );
}
return (
@@ -1745,7 +2834,9 @@ function IndexingMonitor({
Enrich
- {job.error_message && {job.error_message}
}
+ {job.error_message && (
+ {job.error_message}
+ )}
);
})}
@@ -1786,6 +2877,14 @@ const fallbackSetupChecks: SetupCheck[] = [
},
];
+const publicSearchSetupCheckIds = new Set(["env", "project", "schema", "openai"]);
+
+function hasReadyPublicSearchSetup(checks: SetupCheck[]) {
+ return Array.from(publicSearchSetupCheckIds).every(
+ (id) => checks.find((check) => check.id === id)?.status === "ready",
+ );
+}
+
function setupBadgeClasses(status: SetupCheckStatus) {
if (status === "ready") {
return toneSuccess;
@@ -2003,16 +3102,104 @@ function GuideTrigger({ onOpen }: { onOpen: () => void }) {
);
}
+function answerReferencesDocument(answer: RagAnswer | null, documentId: string) {
+ if (!answer) return false;
+ return (
+ answer.citations.some((citation) => citation.document_id === documentId) ||
+ answer.sources.some((source) => source.document_id === documentId) ||
+ Boolean(answer.bestSource?.document_id === documentId) ||
+ Boolean(answer.relatedDocuments?.some((document) => document.document_id === documentId)) ||
+ Boolean(answer.visualEvidence?.some((image) => image.document_id === documentId))
+ );
+}
+
+function applyRenamedDocumentToAnswer(answer: RagAnswer | null, document: ClinicalDocument) {
+ if (!answer || !answerReferencesDocument(answer, document.id)) return answer;
+ const renameCitation = (item: T): T =>
+ item.document_id === document.id ? { ...item, title: document.title } : item;
+ const renameRelated = (item: RelatedDocument): RelatedDocument =>
+ item.document_id === document.id ? { ...item, title: document.title } : item;
+
+ return {
+ ...answer,
+ citations: answer.citations.map(renameCitation),
+ quoteCards: answer.quoteCards?.map(renameCitation),
+ sources: answer.sources.map(renameCitation),
+ visualEvidence: answer.visualEvidence?.map(renameCitation),
+ bestSource: answer.bestSource ? renameCitation(answer.bestSource) : answer.bestSource,
+ relatedDocuments: answer.relatedDocuments?.map(renameRelated),
+ smartPanel: answer.smartPanel
+ ? {
+ ...answer.smartPanel,
+ bestSource: answer.smartPanel.bestSource
+ ? renameCitation(answer.smartPanel.bestSource)
+ : answer.smartPanel.bestSource,
+ relatedDocuments: answer.smartPanel.relatedDocuments?.map(renameRelated),
+ }
+ : answer.smartPanel,
+ } satisfies RagAnswer;
+}
+
+function normalizedPollDelay(value: unknown) {
+ const parsed = typeof value === "number" ? value : Number(value);
+ if (!Number.isFinite(parsed) || parsed <= 0) return null;
+ return Math.min(Math.max(parsed, 3_000), setupRecheckPollMs);
+}
+
+function shorterPollDelay(current: number | null, next: unknown) {
+ const normalized = normalizedPollDelay(next);
+ if (!normalized) return current;
+ return current === null ? normalized : Math.min(current, normalized);
+}
+
+function hasActiveIndexingWork(
+ documents: ClinicalDocument[],
+ jobs: IngestionJob[],
+ batches: ImportBatch[],
+ routeHint = false,
+) {
+ return (
+ routeHint ||
+ documents.some((document) => document.status === "queued" || document.status === "processing") ||
+ jobs.some((job) => job.status === "pending" || job.status === "processing") ||
+ batches.some((batch) => batch.status === "queued" || batch.status === "processing")
+ );
+}
+
+function setupNeedsSlowRecheck(checks: SetupCheck[]) {
+ return checks.some((check) => check.status !== "ready");
+}
+
+function mergeDocumentRefresh(current: ClinicalDocument[], updates: ClinicalDocument[]) {
+ const currentById = new Map(current.map((document) => [document.id, document]));
+ return updates.map((document) => {
+ const existing = currentById.get(document.id);
+ if (!existing) return document;
+ return {
+ ...existing,
+ ...document,
+ labels: document.labels ?? existing.labels,
+ summary: document.summary ?? existing.summary,
+ };
+ });
+}
+
export function ClinicalDashboard() {
const mainRef = useRef(null);
const scrollFrameRef = useRef(null);
const navSyncLockRef = useRef(null);
+ const refreshInFlightRef = useRef | null>(null);
const [documents, setDocuments] = useState([]);
+ const [documentsPagination, setDocumentsPagination] = useState(null);
+ const [loadingMoreDocuments, setLoadingMoreDocuments] = useState(false);
const [jobs, setJobs] = useState([]);
const [batches, setBatches] = useState([]);
const [query, setQuery] = useState("");
+ const [searchMode, setSearchMode] = useState("answer");
const [answer, setAnswer] = useState(null);
const [sources, setSources] = useState([]);
+ const [documentMatches, setDocumentMatches] = useState([]);
+ const [searchRelevance, setSearchRelevance] = useState(null);
const [loading, setLoading] = useState(false);
const [answerProgress, setAnswerProgress] = useState(null);
const [error, setError] = useState(null);
@@ -2020,87 +3207,218 @@ export function ClinicalDashboard() {
const [setupChecks, setSetupChecks] = useState(fallbackSetupChecks);
const [demoMode, setDemoMode] = useState(false);
const [apiUnavailable, setApiUnavailable] = useState(false);
+ const [localProjectReady, setLocalProjectReady] = useState(true);
const [isOnline, setIsOnline] = useState(true);
const [selectedDocumentIds, setSelectedDocumentIds] = useState([]);
const [copiedAction, setCopiedAction] = useState(null);
const [activeHash, setActiveHash] = useState("#search");
const [guideOpen, setGuideOpen] = useState(false);
const [indexingActionId, setIndexingActionId] = useState(null);
+ const [indexingActive, setIndexingActive] = useState(false);
+ const [nextRefreshDelayMs, setNextRefreshDelayMs] = useState(null);
const { theme, toggleTheme } = useTheme();
const auth = useAuthSession();
const { status: authStatus, authorizationHeader, markSessionExpired } = auth;
const supabaseEnvStatus = setupChecks.find((check) => check.id === "env")?.status;
const browserAuthUnavailableDemoFallback = !auth.isConfigured && supabaseEnvStatus !== "ready";
- const clientDemoMode = demoMode || process.env.NEXT_PUBLIC_DEMO_MODE === "true" || browserAuthUnavailableDemoFallback;
- const canUsePrivateApis = clientDemoMode || authStatus === "authenticated";
+ const localNoAuthMode = isLocalNoAuthMode();
+ const clientDemoMode =
+ demoMode || process.env.NEXT_PUBLIC_DEMO_MODE === "true" || browserAuthUnavailableDemoFallback || localNoAuthMode;
+ const uploadReadOnlyMode =
+ demoMode || process.env.NEXT_PUBLIC_DEMO_MODE === "true" || browserAuthUnavailableDemoFallback;
+ const canUsePrivateApis = localProjectReady && (localNoAuthMode || authStatus === "authenticated");
+ const canRunSearch = clientDemoMode || hasReadyPublicSearchSetup(setupChecks);
const openGuide = useCallback(() => setGuideOpen(true), []);
const closeGuide = useCallback(() => setGuideOpen(false), []);
- const refresh = useCallback(async () => {
- setApiUnavailable(false);
- const setupResponse = await fetch("/api/setup-status").catch(() => null);
+ const refresh = useCallback(
+ async (options: RefreshOptions = {}) => {
+ if (refreshInFlightRef.current) {
+ return refreshInFlightRef.current;
+ }
- if (!setupResponse) {
- setApiUnavailable(true);
- setSetupWarning("The local API is unavailable.");
- return;
- }
+ const promise = (async () => {
+ const includeSetup = options.includeSetup ?? true;
+ const includeDashboardData = options.includeDashboardData ?? true;
+ const includeDocumentMeta = options.includeDocumentMeta ?? true;
+ let nextDemoMode = clientDemoMode;
+ let routeIndexingActive = false;
+ let routePollDelayMs: number | null = null;
+
+ setApiUnavailable(false);
+
+ const localIdentity = await readLocalProjectIdentity().catch(() => null);
+ if (!localIdentity?.localServer?.safeLocalOrigin) {
+ setLocalProjectReady(false);
+ setApiUnavailable(true);
+ setSetupWarning(unsafeLocalProjectMessage(localIdentity));
+ setDocuments([]);
+ setDocumentsPagination(null);
+ setJobs([]);
+ setBatches([]);
+ setIndexingActive(false);
+ setNextRefreshDelayMs(null);
+ return;
+ }
+ setLocalProjectReady(true);
+
+ if (includeSetup) {
+ const setupResponse = await fetch("/api/setup-status", { cache: "no-store" }).catch(() => null);
+
+ if (!setupResponse) {
+ setApiUnavailable(true);
+ setSetupWarning("The local API is unavailable.");
+ return;
+ }
+
+ if (setupResponse.ok) {
+ const payload = (await setupResponse.json()) as SetupStatusPayload;
+ setSetupChecks(payload.checks ?? fallbackSetupChecks);
+ nextDemoMode = Boolean(payload.demoMode);
+ routeIndexingActive = Boolean(payload.indexingActive);
+ routePollDelayMs = shorterPollDelay(routePollDelayMs, payload.pollAfterMs);
+ if (nextDemoMode) setDemoMode(true);
+ } else {
+ setApiUnavailable(true);
+ }
+ }
- let nextDemoMode = clientDemoMode;
- if (setupResponse.ok) {
- const payload = await setupResponse.json();
- setSetupChecks(payload.checks ?? fallbackSetupChecks);
- nextDemoMode = Boolean(payload.demoMode);
- if (nextDemoMode) setDemoMode(true);
- if (!nextDemoMode && authStatus !== "authenticated") {
- setDocuments([]);
- setJobs([]);
- setBatches([]);
- return;
- }
- }
+ if (!nextDemoMode && !canUsePrivateApis) {
+ setDocuments([]);
+ setDocumentsPagination(null);
+ setJobs([]);
+ setBatches([]);
+ setIndexingActive(routeIndexingActive);
+ setNextRefreshDelayMs(routePollDelayMs);
+ return;
+ }
+
+ if (!includeDashboardData) {
+ setIndexingActive(routeIndexingActive);
+ setNextRefreshDelayMs(routePollDelayMs);
+ return;
+ }
- const protectedHeaders = nextDemoMode ? undefined : authorizationHeader;
- const [documentsResponse, jobsResponse, batchesResponse] = await Promise.all([
- fetch("/api/documents", { headers: protectedHeaders }),
- fetch("/api/ingestion/jobs", { headers: protectedHeaders }),
- fetch("/api/ingestion/batches", { headers: protectedHeaders }),
- ]);
+ const protectedHeaders = nextDemoMode ? undefined : authorizationHeader;
+ const documentParams = new URLSearchParams({ limit: String(documentPageSize) });
+ if (!includeDocumentMeta) {
+ documentParams.set("includeMeta", "false");
+ }
- if (documentsResponse.status === 401 || jobsResponse.status === 401 || batchesResponse.status === 401) {
- markSessionExpired();
- setDocuments([]);
- setJobs([]);
- setBatches([]);
- return;
- }
+ const [documentsResponse, jobsResponse, batchesResponse] = await Promise.all([
+ fetch(`/api/documents?${documentParams.toString()}`, { headers: protectedHeaders }),
+ fetch("/api/ingestion/jobs", { headers: protectedHeaders }),
+ fetch("/api/ingestion/batches", { headers: protectedHeaders }),
+ ]);
- if (documentsResponse.ok) {
- const payload = await documentsResponse.json();
- setDocuments(payload.documents ?? []);
- if (payload.demoMode) setDemoMode(true);
- if (payload.setupRequired) setSetupWarning(payload.error);
- } else {
- setApiUnavailable(true);
- }
+ if (documentsResponse.status === 401 || jobsResponse.status === 401 || batchesResponse.status === 401) {
+ markSessionExpired();
+ setDocuments([]);
+ setDocumentsPagination(null);
+ setJobs([]);
+ setBatches([]);
+ setIndexingActive(false);
+ setNextRefreshDelayMs(null);
+ return;
+ }
+
+ let nextDocuments: ClinicalDocument[] = [];
+ let nextJobs: IngestionJob[] = [];
+ let nextBatches: ImportBatch[] = [];
+
+ if (documentsResponse.ok) {
+ const payload = (await documentsResponse.json()) as DocumentsPayload;
+ nextDocuments = payload.documents ?? [];
+ setDocuments((current) =>
+ includeDocumentMeta ? nextDocuments : mergeDocumentRefresh(current, nextDocuments),
+ );
+ setDocumentsPagination(payload.pagination ?? null);
+ routeIndexingActive ||= Boolean(payload.indexing?.active);
+ routePollDelayMs = shorterPollDelay(routePollDelayMs, payload.indexing?.pollAfterMs);
+ if (payload.demoMode) setDemoMode(true);
+ if (payload.setupRequired) setSetupWarning(payload.error ?? null);
+ } else {
+ setApiUnavailable(true);
+ }
+
+ if (jobsResponse.ok) {
+ const payload = (await jobsResponse.json()) as JobsPayload;
+ nextJobs = payload.jobs ?? [];
+ setJobs(nextJobs);
+ routeIndexingActive ||= Boolean(payload.hasActiveJobs);
+ routePollDelayMs = shorterPollDelay(routePollDelayMs, payload.pollAfterMs);
+ if (payload.demoMode) setDemoMode(true);
+ if (payload.setupRequired) setSetupWarning(payload.error ?? null);
+ } else {
+ setApiUnavailable(true);
+ }
+
+ if (batchesResponse.ok) {
+ const payload = (await batchesResponse.json()) as BatchesPayload;
+ nextBatches = payload.batches ?? [];
+ setBatches(nextBatches);
+ routeIndexingActive ||= Boolean(payload.hasActiveBatches);
+ routePollDelayMs = shorterPollDelay(routePollDelayMs, payload.pollAfterMs);
+ if (payload.demoMode) setDemoMode(true);
+ } else {
+ setApiUnavailable(true);
+ }
- if (jobsResponse.ok) {
- const payload = await jobsResponse.json();
- setJobs(payload.jobs ?? []);
- if (payload.demoMode) setDemoMode(true);
- if (payload.setupRequired) setSetupWarning(payload.error);
- } else {
- setApiUnavailable(true);
+ const activeWork = hasActiveIndexingWork(nextDocuments, nextJobs, nextBatches, routeIndexingActive);
+ setIndexingActive(activeWork);
+ setNextRefreshDelayMs(routePollDelayMs ?? (activeWork ? activeIndexingPollFallbackMs : null));
+ })();
+
+ refreshInFlightRef.current = promise;
+ try {
+ return await promise;
+ } finally {
+ if (refreshInFlightRef.current === promise) {
+ refreshInFlightRef.current = null;
+ }
+ }
+ },
+ [authorizationHeader, canUsePrivateApis, clientDemoMode, markSessionExpired],
+ );
+
+ const loadMoreDocuments = useCallback(async () => {
+ if (!documentsPagination?.hasMore || loadingMoreDocuments || !canUsePrivateApis) {
+ return;
}
- if (batchesResponse.ok) {
- const payload = await batchesResponse.json();
- setBatches(payload.batches ?? []);
- if (payload.demoMode) setDemoMode(true);
- } else {
- setApiUnavailable(true);
+ setLoadingMoreDocuments(true);
+ try {
+ const protectedHeaders = clientDemoMode ? undefined : authorizationHeader;
+ const response = await fetch(
+ `/api/documents?limit=${documentPageSize}&offset=${documentsPagination.nextOffset}`,
+ { headers: protectedHeaders },
+ );
+ if (response.status === 401) {
+ markSessionExpired();
+ return;
+ }
+ if (!response.ok) {
+ setApiUnavailable(true);
+ return;
+ }
+ const payload = await response.json();
+ const nextDocuments = (payload.documents ?? []) as ClinicalDocument[];
+ setDocuments((current) => {
+ const seen = new Set(current.map((document) => document.id));
+ return [...current, ...nextDocuments.filter((document) => !seen.has(document.id))];
+ });
+ setDocumentsPagination(payload.pagination ?? null);
+ } finally {
+ setLoadingMoreDocuments(false);
}
- }, [authStatus, authorizationHeader, clientDemoMode, markSessionExpired]);
+ }, [
+ authorizationHeader,
+ canUsePrivateApis,
+ clientDemoMode,
+ documentsPagination,
+ loadingMoreDocuments,
+ markSessionExpired,
+ ]);
const retryJob = useCallback(
async (jobId: string) => {
@@ -2114,7 +3432,7 @@ export function ClinicalDashboard() {
markSessionExpired();
return;
}
- await refresh();
+ await refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false });
} finally {
setIndexingActionId(null);
}
@@ -2138,7 +3456,7 @@ export function ClinicalDashboard() {
markSessionExpired();
return;
}
- await refresh();
+ await refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false });
} finally {
setIndexingActionId(null);
}
@@ -2150,24 +3468,92 @@ export function ClinicalDashboard() {
[reindexDocument],
);
+ const handleDocumentRenamed = useCallback((updatedDocument: ClinicalDocument) => {
+ setDocuments((current) =>
+ current.map((document) => (document.id === updatedDocument.id ? { ...document, ...updatedDocument } : document)),
+ );
+ setSources((current) =>
+ current.map((source) =>
+ source.document_id === updatedDocument.id ? { ...source, title: updatedDocument.title } : source,
+ ),
+ );
+ setDocumentMatches((current) =>
+ current.map((document) =>
+ document.document_id === updatedDocument.id ? { ...document, title: updatedDocument.title } : document,
+ ),
+ );
+ setAnswer((current) => applyRenamedDocumentToAnswer(current, updatedDocument));
+ }, []);
+
+ const handleDocumentDeleted = useCallback(
+ (result: DocumentDeleteResult) => {
+ setDocuments((current) => current.filter((document) => document.id !== result.documentId));
+ setSelectedDocumentIds((current) => current.filter((documentId) => documentId !== result.documentId));
+ setSources((current) => current.filter((source) => source.document_id !== result.documentId));
+ setDocumentMatches((current) => current.filter((document) => document.document_id !== result.documentId));
+ setAnswer((current) => (answerReferencesDocument(current, result.documentId) ? null : current));
+ if (result.storageWarnings.length > 0) {
+ setError(`Document deleted, but storage cleanup needs review: ${result.storageWarnings.join("; ")}`);
+ }
+ void refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false }).catch(
+ () => undefined,
+ );
+ },
+ [refresh],
+ );
+
+ const activeIndexingWork = useMemo(
+ () => hasActiveIndexingWork(documents, jobs, batches, indexingActive),
+ [batches, documents, indexingActive, jobs],
+ );
+ const needsSetupRecheck = useMemo(() => setupNeedsSlowRecheck(setupChecks), [setupChecks]);
+
+ useEffect(() => {
+ refresh({ includeSetup: true, includeDashboardData: true, includeDocumentMeta: true }).catch(() => undefined);
+ }, [authStatus, authorizationHeader, clientDemoMode, refresh]);
+
+ useEffect(() => {
+ const hasScheduledWork = activeIndexingWork || needsSetupRecheck;
+ if (!shouldPollForUpdates(demoMode, document.visibilityState, hasScheduledWork)) {
+ return;
+ }
+
+ const delay = activeIndexingWork ? (nextRefreshDelayMs ?? activeIndexingPollFallbackMs) : setupRecheckPollMs;
+ const timeout = window.setTimeout(() => {
+ if (!shouldPollForUpdates(demoMode, document.visibilityState, hasScheduledWork)) {
+ return;
+ }
+
+ refresh({
+ includeSetup: !activeIndexingWork,
+ includeDashboardData: activeIndexingWork,
+ includeDocumentMeta: false,
+ }).catch(() => undefined);
+ }, delay);
+
+ return () => window.clearTimeout(timeout);
+ }, [activeIndexingWork, demoMode, needsSetupRecheck, nextRefreshDelayMs, refresh]);
+
useEffect(() => {
- const tick = () => {
- if (shouldPollForUpdates(demoMode, document.visibilityState)) {
- refresh().catch(() => undefined);
+ const refreshVisibleDashboard = () => {
+ if (document.visibilityState !== "visible") {
+ return;
}
+
+ refresh({
+ includeSetup: true,
+ includeDashboardData: activeIndexingWork || canUsePrivateApis || clientDemoMode,
+ includeDocumentMeta: false,
+ }).catch(() => undefined);
};
- const initial = window.setTimeout(() => {
- refresh().catch(() => undefined);
- }, 0);
- const interval = window.setInterval(tick, 7000);
- document.addEventListener("visibilitychange", tick);
+ document.addEventListener("visibilitychange", refreshVisibleDashboard);
+ window.addEventListener("focus", refreshVisibleDashboard);
return () => {
- window.clearTimeout(initial);
- window.clearInterval(interval);
- document.removeEventListener("visibilitychange", tick);
+ document.removeEventListener("visibilitychange", refreshVisibleDashboard);
+ window.removeEventListener("focus", refreshVisibleDashboard);
};
- }, [authStatus, authorizationHeader, clientDemoMode, demoMode, refresh]);
+ }, [activeIndexingWork, canUsePrivateApis, clientDemoMode, refresh]);
useEffect(() => {
const updateOnline = () => setIsOnline(navigator.onLine);
@@ -2201,39 +3587,191 @@ export function ClinicalDashboard() {
};
}, []);
+ async function requestDocuments(queryText: string) {
+ const response = await fetch("/api/search", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...(clientDemoMode ? {} : authorizationHeader),
+ },
+ body: JSON.stringify({
+ query: queryText,
+ mode: "documents",
+ documentIds: selectedDocumentIds.length > 0 ? selectedDocumentIds : undefined,
+ documentLimit: 30,
+ topK: 20,
+ }),
+ });
+
+ if (response.status === 401) {
+ markSessionExpired();
+ throw makeSearchError("Search request was not authorized by the server.", 401, false);
+ }
+ if (!response.ok) {
+ const payload = await response.json().catch(() => ({}));
+ const message = typeof payload?.error === "string" ? payload.error : "Document search failed";
+ throw makeSearchError(message, response.status, isRetryableStatus(response.status));
+ }
+ const payload = await response.json();
+ if (payload.demoMode) setDemoMode(true);
+
+ return {
+ kind: "documents" as const,
+ query: queryText,
+ sources: (payload.results ?? []) as SearchResult[],
+ documentMatches: (payload.documentMatches ?? []) as DocumentMatch[],
+ relevance: payload.relevance as EvidenceRelevance | undefined,
+ demoMode: payload.demoMode,
+ };
+ }
+
+ async function requestAnswer(queryText: string) {
+ const response = await fetch("/api/answer/stream", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...(clientDemoMode ? {} : authorizationHeader),
+ },
+ body: JSON.stringify({
+ query: queryText,
+ documentIds: selectedDocumentIds.length > 0 ? selectedDocumentIds : undefined,
+ }),
+ });
+
+ if (response.status === 401) {
+ markSessionExpired();
+ throw makeSearchError("Search request was not authorized by the server.", 401, false);
+ }
+ if (!response.ok) {
+ const payload = await response.json().catch(() => ({}));
+ const message = typeof payload?.error === "string" ? payload.error : "Answer generation failed";
+ throw makeSearchError(message, response.status, isRetryableStatus(response.status));
+ }
+
+ const payload = await readAnswerStream(response, setAnswerProgress);
+ return {
+ kind: "answer" as const,
+ query: queryText,
+ payload,
+ };
+ }
+
+ async function runWithRetries(operation: () => Promise) {
+ let lastError: unknown;
+ for (let attempt = 0; attempt <= searchRetryCount; attempt += 1) {
+ try {
+ return await operation();
+ } catch (error) {
+ lastError = error;
+ if (!isRetryableError(error) || attempt >= searchRetryCount) break;
+
+ const message = progressForRetry(attempt + 1);
+ setAnswerProgress(message);
+ await sleep(searchRetryDelaysMs[attempt] ?? searchRetryDelaysMs[searchRetryDelaysMs.length - 1]);
+ }
+ }
+ throw lastError;
+ }
+
+ function resultUsable(payload: SearchResultModePayload) {
+ if (payload.kind === "documents") {
+ return payload.sources.length > 0 || payload.documentMatches.length > 0;
+ }
+ return answerPayloadIsUsable(payload.payload);
+ }
+
+ function applySearchResult(payload: SearchResultModePayload) {
+ if (payload.kind === "documents") {
+ setDocumentMatches(payload.documentMatches);
+ setSources(payload.sources);
+ setSearchRelevance(payload.relevance ?? null);
+ return;
+ }
+
+ const answerData = payload.payload;
+ setAnswer(answerData);
+ setSources(answerData.sources ?? []);
+ setSearchRelevance(answerData.relevance ?? answerData.smartPanel?.relevance ?? null);
+ setDocumentMatches(
+ answerData.relatedDocuments?.map((document) => ({
+ document_id: document.document_id,
+ title: document.title,
+ file_name: document.file_name,
+ labels: document.labels,
+ summarySnippet: document.summary,
+ bestPages: document.best_pages,
+ bestChunkIds: document.best_chunk_ids,
+ imageCount: document.image_count,
+ tableCount: document.table_count ?? 0,
+ matchReason: document.match_reason,
+ score: document.score,
+ })) ?? [],
+ );
+ if (answerData.demoMode) setDemoMode(true);
+ }
+
async function ask() {
- if (query.trim().length < 2) return;
- if (!clientDemoMode && authStatus !== "authenticated") {
- setError("Sign in before searching private guideline documents.");
+ const trimmedQuery = query.trim();
+ if (!trimmedQuery) return;
+ if (!canRunSearch) {
+ setError("Search setup is not ready.");
return;
}
+
setLoading(true);
setError(null);
- setAnswerProgress("Searching indexed documents.");
+ setSearchRelevance(null);
+ setAnswerProgress(searchMode === "documents" ? "Finding matching documents." : "Searching indexed documents.");
+
+ const fallbackQuery = keywordQueryFromNaturalLanguage(trimmedQuery);
+ const queryPlan =
+ fallbackQuery && fallbackQuery !== trimmedQuery
+ ? [
+ { query: trimmedQuery, isKeyword: false },
+ { query: fallbackQuery, isKeyword: true },
+ ]
+ : [{ query: trimmedQuery, isKeyword: false }];
try {
- const response = await fetch("/api/answer/stream", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- ...(clientDemoMode ? {} : authorizationHeader),
- },
- body: JSON.stringify({
- query,
- documentIds: selectedDocumentIds.length > 0 ? selectedDocumentIds : undefined,
- }),
- });
- if (response.status === 401) markSessionExpired();
- if (!response.ok) {
- const payload = await response.json().catch(() => ({}));
- throw new Error(payload.error || "Answer generation failed");
+ let successfulPayload: SearchResultModePayload | null = null;
+ let lastError: SearchError | null = null;
+
+ for (const entry of queryPlan) {
+ if (entry.isKeyword) setAnswerProgress("Trying keyword-based search...");
+
+ try {
+ const payload =
+ searchMode === "documents"
+ ? await runWithRetries(() => requestDocuments(entry.query))
+ : await runWithRetries(() => requestAnswer(entry.query));
+
+ if (!resultUsable(payload)) {
+ lastError = makeSearchError("No usable results were found.", 404, false);
+ if (!entry.isKeyword) {
+ continue;
+ }
+ break;
+ }
+
+ successfulPayload = payload;
+ break;
+ } catch (requestError) {
+ lastError = requestError as SearchError;
+ if (queryPlan.length > 1 && !entry.isKeyword) {
+ continue;
+ }
+ throw requestError;
+ }
+ }
+
+ if (!successfulPayload) {
+ if (lastError) throw lastError;
+ throw new Error("Search did not return usable results.");
}
- const payload = await readAnswerStream(response, setAnswerProgress);
- setAnswer(payload);
- setSources(payload.sources ?? []);
- if (payload.demoMode) setDemoMode(true);
+
+ applySearchResult(successfulPayload);
} catch (requestError) {
- setError(requestError instanceof Error ? requestError.message : "Answer generation failed");
+ setError(requestError instanceof Error ? requestError.message : "Search failed");
} finally {
setLoading(false);
setAnswerProgress(null);
@@ -2250,6 +3788,12 @@ export function ClinicalDashboard() {
setSelectedDocumentIds([documentId]);
}
+ function answerFromDocument(documentId: string) {
+ setSelectedDocumentIds([documentId]);
+ setSearchMode("answer");
+ window.requestAnimationFrame(() => mainRef.current?.scrollTo({ top: 0, behavior: "smooth" }));
+ }
+
function followUpFromQuote(quote: QuoteCard) {
setQuery(createQuoteFollowUp(quote));
setSelectedDocumentIds([quote.document_id]);
@@ -2348,13 +3892,49 @@ export function ClinicalDashboard() {
() => answer?.relatedDocuments ?? answer?.smartPanel?.relatedDocuments ?? [],
[answer],
);
+ const currentRelevance = answer?.relevance ?? answer?.smartPanel?.relevance ?? searchRelevance;
+ const weakEvidence = isWeakRelevance(currentRelevance);
const safetyFindings = useMemo(() => extractSafetyFindings(answer), [answer]);
const bestSource = answer?.bestSource ?? answer?.smartPanel?.bestSource ?? null;
const sourceSummary = answer?.evidenceSummary ?? answer?.smartPanel?.evidenceSummary;
const gaps = answer?.conflictsOrGaps ?? answer?.smartPanel?.conflictsOrGaps ?? [];
+ const sourceLookup = useMemo(() => new Map(sources.map((source) => [source.id, source])), [sources]);
+ const safeAnswerText = useMemo(() => sanitizeAnswerDisplayText(answer?.answer ?? ""), [answer?.answer]);
+ const safeAnswerSections = useMemo(() => {
+ return (answer?.answerSections ?? [])
+ .map((section) => {
+ const heading = sanitizeDisplayText(section.heading, { minLength: 1, minTokens: 1 });
+ const body = sanitizeAnswerDisplayText(section.body, { minLength: 8, minTokens: 2 });
+ if (!heading || !body) return null;
+
+ const citationSources: SearchResult[] = [];
+ const seenCitationIds = new Set();
+ for (const id of section.citation_chunk_ids) {
+ if (seenCitationIds.has(id)) continue;
+ const source = sourceLookup.get(id);
+ if (!source) continue;
+ seenCitationIds.add(id);
+ citationSources.push(source);
+ }
+
+ return {
+ ...section,
+ heading,
+ body,
+ citationSources,
+ };
+ })
+ .filter((section): section is AnswerSection & { citationSources: SearchResult[] } => section !== null);
+ }, [answer?.answerSections, sourceLookup]);
+
const showSystemNotice = demoMode || setupWarning;
const bottomNavItems = [
- { label: "Answer", icon: Search, href: "#search", count: null },
+ {
+ label: searchMode === "answer" ? "Answer" : "Docs",
+ icon: searchMode === "answer" ? Search : FileText,
+ href: "#search",
+ count: searchMode === "documents" ? documentMatches.length : null,
+ },
{ label: "Quotes", icon: Quote, href: "#quotes", count: answer ? (answer.quoteCards ?? []).length : null },
{ label: "Images", icon: FileImage, href: "#images", count: answer ? visualEvidence.length : null },
{ label: "Sources", icon: FileText, href: "#sources", count: answer ? sources.length : null },
@@ -2376,7 +3956,7 @@ export function ClinicalDashboard() {
);
- const showAuthPanel = !clientDemoMode && authStatus !== "authenticated";
+ const showAuthPanel = !clientDemoMode && !canUsePrivateApis;
const showDegradedNotice = !isOnline || apiUnavailable;
const renderDegradedNotice = () => (
setQuery("")}
onClearScope={() => setSelectedDocumentIds([])}
@@ -2435,17 +4017,24 @@ export function ClinicalDashboard() {
) : null
}
@@ -2470,21 +4059,61 @@ export function ClinicalDashboard() {
)}
- {loading && !answer ? (
+ {searchMode === "documents" ? (
+
+ ) : loading && !answer ? (
) : answer ? (
-
-
-
-
+
+
+
copyText("ward-note-panel", formatWardNote(answer, demoMode))}
+ collapsed={weakEvidence}
+ />
+
+
+
+
+
+
+
+ Source narrative
+
+ Full source-linked answer text and section citations
+
+
+
+
+
+
+
+
+
@@ -2492,6 +4121,7 @@ export function ClinicalDashboard() {
copyText("answer", formatAnswerForClipboard(answer))}
onCopyDraft={() => copyText("ward-note", formatWardNote(answer, demoMode))}
/>
@@ -2533,7 +4163,7 @@ export function ClinicalDashboard() {
{showSystemNotice && answer ? renderSystemNotice("sm:hidden") : null}
- {answer && (
+ {searchMode === "answer" && answer && (
)}
- {answer && }
- {answer && }
- {answer && (
- copyText("ward-note-panel", formatWardNote(answer, demoMode))}
- />
+ {searchMode === "answer" && answer && }
+ {searchMode === "answer" && answer && (
+
)}
-
- {bestSource ? (
+ {searchMode === "answer" && bestSource ? (
) : null}
- {answer?.answerSections?.length ? (
+ {searchMode === "answer" && safeAnswerSections.length > 0 ? (
- {answer.answerSections.map((section) => (
-
- {section.heading}
-
-
-
-
- ))}
+ {safeAnswerSections.map((section) => {
+ const sectionContent = sectionBodyContent(section.body);
+ return (
+
+
+
+
+
+
+
+
+ {section.heading}
+
+
+ {section.citationSources.length} linked source passage
+ {section.citationSources.length === 1 ? "" : "s"}
+
+
+
+
+
+
+
+ {section.citationSources.length > 0 ? (
+
+ ) : (
+ No linked passage metadata
+ )}
+
+
+ );
+ })}
+ ) : searchMode === "answer" ? (
+
+
+
) : null}
@@ -2664,12 +4335,14 @@ export function ClinicalDashboard() {
Developer setup status
- {!clientDemoMode && authStatus !== "authenticated" && }
+ {showAuthPanel && }
Clinical upload
{
+ void refresh({ includeSetup: false, includeDashboardData: true, includeDocumentMeta: false });
+ }}
+ demoMode={uploadReadOnlyMode}
+ canUpload={canUsePrivateApis}
authorizationHeader={authorizationHeader}
/>
diff --git a/src/components/DocumentManagementActions.tsx b/src/components/DocumentManagementActions.tsx
new file mode 100644
index 000000000..1b2db1690
--- /dev/null
+++ b/src/components/DocumentManagementActions.tsx
@@ -0,0 +1,249 @@
+"use client";
+
+import { AlertTriangle, Check, Loader2, Pencil, Trash2, X } from "lucide-react";
+import { FormEvent, useId, useState } from "react";
+import {
+ cn,
+ fieldControlPlain,
+ fieldLabel,
+ floatingControl,
+ primaryControl,
+ textMuted,
+ toneDanger,
+ toolbarButton,
+} from "@/components/ui-primitives";
+import { useAuthSession } from "@/lib/supabase/client";
+import type { ClinicalDocument } from "@/lib/types";
+
+type ManagedDocument = Pick
;
+
+export type DocumentDeleteResult = {
+ deleted: true;
+ documentId: string;
+ storageRemoved: number;
+ storageWarnings: string[];
+};
+
+type DialogMode = "rename" | "delete" | null;
+
+export function DocumentManagementActions({
+ document,
+ disabled = false,
+ className,
+ onRenamed,
+ onDeleted,
+}: {
+ document: ManagedDocument;
+ disabled?: boolean;
+ className?: string;
+ onRenamed?: (document: ClinicalDocument) => void;
+ onDeleted?: (result: DocumentDeleteResult) => void;
+}) {
+ const titleId = useId();
+ const { status: authStatus, authorizationHeader, markSessionExpired } = useAuthSession();
+ const [mode, setMode] = useState(null);
+ const [title, setTitle] = useState(document.title);
+ const [deleteConfirmation, setDeleteConfirmation] = useState("");
+ const [error, setError] = useState(null);
+ const [pending, setPending] = useState(false);
+ const canManage = !disabled && authStatus === "authenticated";
+
+ function resetDialogState() {
+ setTitle(document.title);
+ setDeleteConfirmation("");
+ setError(null);
+ }
+
+ function openDialog(nextMode: Exclude) {
+ resetDialogState();
+ setMode(nextMode);
+ }
+
+ function closeDialog() {
+ if (pending) return;
+ setMode(null);
+ resetDialogState();
+ }
+
+ async function readPayload(response: Response) {
+ const payload = (await response.json().catch(() => ({}))) as Record;
+ if (response.status === 401) markSessionExpired();
+ if (!response.ok) {
+ const message = typeof payload.error === "string" ? payload.error : "Document action failed.";
+ throw new Error(message);
+ }
+ return payload;
+ }
+
+ async function submitRename(event: FormEvent) {
+ event.preventDefault();
+ const nextTitle = title.trim();
+ if (!nextTitle) {
+ setError("Enter a document title.");
+ return;
+ }
+ setPending(true);
+ setError(null);
+ try {
+ const response = await fetch(`/api/documents/${document.id}`, {
+ method: "PATCH",
+ headers: {
+ ...authorizationHeader,
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({ title: nextTitle }),
+ });
+ const payload = await readPayload(response);
+ if (payload.document) onRenamed?.(payload.document as ClinicalDocument);
+ setMode(null);
+ } catch (renameError) {
+ setError(renameError instanceof Error ? renameError.message : "Document rename failed.");
+ } finally {
+ setPending(false);
+ }
+ }
+
+ async function submitDelete(event: FormEvent) {
+ event.preventDefault();
+ if (deleteConfirmation !== document.title) {
+ setError("Type the current document title to confirm permanent deletion.");
+ return;
+ }
+ setPending(true);
+ setError(null);
+ try {
+ const response = await fetch(`/api/documents/${document.id}`, {
+ method: "DELETE",
+ headers: authorizationHeader,
+ });
+ const payload = (await readPayload(response)) as DocumentDeleteResult;
+ onDeleted?.(payload);
+ setMode(null);
+ } catch (deleteError) {
+ setError(deleteError instanceof Error ? deleteError.message : "Document delete failed.");
+ } finally {
+ setPending(false);
+ }
+ }
+
+ return (
+ <>
+
+
openDialog("rename")}
+ disabled={!canManage || pending}
+ title="Rename document"
+ aria-label={`Rename ${document.title}`}
+ >
+
+
+
openDialog("delete")}
+ disabled={!canManage || pending}
+ title="Permanently delete document"
+ aria-label={`Permanently delete ${document.title}`}
+ >
+
+
+
+
+ {mode && (
+
+
+
+
+
+ {mode === "rename" ? "Rename document" : "Permanently delete document"}
+
+
+ {mode === "rename"
+ ? "The original file name and storage path will stay unchanged."
+ : "This removes the source, extracted evidence, images, labels, summaries, and related query logs."}
+
+
+
+
+
+
+
+ {error &&
{error}
}
+
+ {mode === "rename" ? (
+
+ ) : (
+
+ )}
+
+
+ )}
+ >
+ );
+}
diff --git a/src/components/DocumentViewer.tsx b/src/components/DocumentViewer.tsx
index d86fc05fc..cf193fa40 100644
--- a/src/components/DocumentViewer.tsx
+++ b/src/components/DocumentViewer.tsx
@@ -3,12 +3,14 @@
/* eslint-disable @next/next/no-img-element */
import Link from "next/link";
+import { useRouter } from "next/navigation";
import {
AlertCircle,
ArrowLeft,
ChevronLeft,
ChevronRight,
ChevronDown,
+ Minimize2,
ExternalLink,
FileImage,
FileText,
@@ -19,8 +21,10 @@ import {
Quote,
RefreshCw,
Sparkles,
+ Target,
} from "lucide-react";
import { useEffect, useRef, useState } from "react";
+import { AccessibleTable } from "@/components/AccessibleTable";
import type { PDFDocumentProxy, RenderTask } from "pdfjs-dist";
import {
appBackdrop,
@@ -44,9 +48,18 @@ import {
} from "@/components/ui-primitives";
import { clearCachedSignedUrl, getCachedSignedUrl, setCachedSignedUrl } from "@/lib/signed-url-cache";
import { formatClinicalDate, normalizeSourceMetadata, sourceStatusLabel } from "@/lib/source-metadata";
+import { isLocalNoAuthMode } from "@/lib/env";
import { useAuthSession } from "@/lib/supabase/client";
import { SafeBoldText } from "@/components/SafeBoldText";
+import { DocumentManagementActions } from "@/components/DocumentManagementActions";
import type { ClinicalDocument, RagAnswer } from "@/lib/types";
+import {
+ cleanClinicalSummaryText,
+ sourceTextForDocumentViewer,
+ sourceTextForIndexedPage,
+} from "@/lib/source-text-sanitizer";
+import { smartEvidenceTags } from "@/lib/evidence-tags";
+import { parseIndexedSourceText } from "@/lib/indexed-source-formatting";
type PageRow = {
id: string;
@@ -55,13 +68,47 @@ type PageRow = {
ocr_used: boolean;
};
+type LocalProjectIdentityPayload = {
+ localServer?: {
+ projectPortStart?: number;
+ projectPortEnd?: number;
+ safeLocalOrigin?: boolean;
+ };
+};
+
+async function readLocalProjectIdentity() {
+ const response = await fetch("/api/local-project-id", { cache: "no-store" });
+ if (!response.ok) return null;
+ return (await response.json()) as LocalProjectIdentityPayload;
+}
+
+function unsafeLocalProjectMessage(identity: LocalProjectIdentityPayload | null) {
+ const range =
+ typeof identity?.localServer?.projectPortStart === "number" &&
+ typeof identity.localServer.projectPortEnd === "number"
+ ? ` Use the URL printed by npm run ensure; managed ports are ${identity.localServer.projectPortStart}-${identity.localServer.projectPortEnd}.`
+ : " Use the URL printed by npm run ensure.";
+ return `This tab is not using the guarded Clinical KB local URL.${range}`;
+}
+
type ImageRow = {
id: string;
page_number: number | null;
caption: string;
image_type?: string | null;
+ searchable?: boolean | null;
clinical_relevance_score?: number | null;
labels?: string[] | null;
+ source_kind?: string | null;
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableTextSnippet?: string | null;
+ clinicalUseClass?: string | null;
+ clinicalUseReason?: string | null;
+ accessibleTableMarkdown?: string | null;
+ tableRows?: string[][] | null;
+ tableColumns?: string[] | null;
};
type ChunkRow = {
@@ -88,16 +135,56 @@ function SourceMetadataSummary({ metadata }: { metadata?: unknown }) {
);
}
+function looksLikeTableText(value?: string | null) {
+ return Boolean(value?.includes("|") && value.split("|").filter((cell) => cell.trim()).length >= 3);
+}
+
function DocumentImage({ image }: { image: ImageRow }) {
const endpoint = `/api/images/${image.id}/signed-url`;
const [url, setUrl] = useState(() => getCachedSignedUrl(endpoint)?.url ?? null);
const [failed, setFailed] = useState(false);
const [attempt, setAttempt] = useState(0);
+ const [shouldLoad, setShouldLoad] = useState(() => Boolean(getCachedSignedUrl(endpoint)));
+ const figureRef = useRef(null);
const { authorizationHeader, markSessionExpired } = useAuthSession();
useEffect(() => {
+ if (shouldLoad) return () => undefined;
+
+ const element = figureRef.current;
+ if (!element || !("IntersectionObserver" in window)) {
+ setShouldLoad(true);
+ return () => undefined;
+ }
+
+ const observer = new IntersectionObserver(
+ (entries) => {
+ if (entries.some((entry) => entry.isIntersecting)) {
+ setShouldLoad(true);
+ observer.disconnect();
+ }
+ },
+ { rootMargin: "640px 0px" },
+ );
+ observer.observe(element);
+ return () => observer.disconnect();
+ }, [shouldLoad]);
+
+ useEffect(() => {
+ if (!shouldLoad) return () => undefined;
+
const cached = getCachedSignedUrl(endpoint);
- if (cached) return () => undefined;
+ if (cached) {
+ let active = true;
+ window.requestAnimationFrame(() => {
+ if (!active) return;
+ setUrl(cached.url);
+ setFailed(false);
+ });
+ return () => {
+ active = false;
+ };
+ }
let active = true;
fetch(endpoint, { headers: authorizationHeader })
@@ -120,12 +207,13 @@ function DocumentImage({ image }: { image: ImageRow }) {
return () => {
active = false;
};
- }, [attempt, authorizationHeader, endpoint, markSessionExpired]);
+ }, [attempt, authorizationHeader, endpoint, markSessionExpired, shouldLoad]);
function retryImage() {
clearCachedSignedUrl(endpoint);
setUrl(null);
setFailed(false);
+ setShouldLoad(true);
setAttempt((current) => current + 1);
}
@@ -134,11 +222,29 @@ function DocumentImage({ image }: { image: ImageRow }) {
setFailed(true);
}
+ const tableHeading = [image.tableLabel, image.tableTitle].filter(Boolean).join(": ");
+ const tableMarkdown = image.accessibleTableMarkdown?.trim()
+ ? image.accessibleTableMarkdown
+ : looksLikeTableText(image.tableTextSnippet)
+ ? image.tableTextSnippet
+ : null;
+ const hasStructuredTable = Boolean(
+ tableMarkdown || image.tableRows?.length || image.tableColumns?.length,
+ );
+ const displayLabels = smartEvidenceTags(
+ image.labels,
+ [tableHeading, image.caption, image.tableTextSnippet].filter(Boolean).join(" "),
+ );
+
return (
-
+
page {image.page_number ?? "n/a"}
{image.image_type ? ` · ${image.image_type.replaceAll("_", " ")}` : ""}
+ {image.tableRole ? ` · ${image.tableRole}` : ""}
+ {image.clinicalUseClass && image.clinicalUseClass !== "clinical_evidence"
+ ? ` · ${image.clinicalUseClass.replaceAll("_", " ")}`
+ : ""}
{failed ? (
@@ -164,17 +270,37 @@ function DocumentImage({ image }: { image: ImageRow }) {
onError={handleImageError}
className="max-h-52 w-full rounded-lg object-contain"
/>
- ) : (
+ ) : shouldLoad ? (
Loading image
+ ) : (
+
+ Image preview will load when visible
+
)}
- {image.caption}
- {image.labels?.length ? (
+
+ {tableHeading && {tableHeading}
}
+ {image.caption}
+
+ {!hasStructuredTable && image.tableTextSnippet ? (
+ {image.tableTextSnippet}
+ ) : null}
+ {image.clinicalUseClass && image.clinicalUseClass !== "clinical_evidence" && image.clinicalUseReason ? (
+ {image.clinicalUseReason}
+ ) : null}
+
+ {displayLabels.length ? (
- {image.labels.slice(0, 5).map((label) => (
+ {displayLabels.map((label) => (
+
+ {anchors.map((anchor) => (
+
+ {anchor.label}
+
+ ))}
+
+ );
+}
+
+function PinnedSourceEvidence({
+ loading,
+ chunk,
+ compact = false,
+ sectionId = "source-evidence",
+}: {
+ loading: boolean;
+ chunk: ChunkRow | undefined;
+ compact?: boolean;
+ sectionId?: "source-evidence" | "source-evidence-rail";
+}) {
+ const displayContent = chunk ? sourceTextForDocumentViewer(chunk.content) : "";
+ const previewLimit = 300;
+ const [expandedChunkId, setExpandedChunkId] = useState(null);
+ const isLong = displayContent.length > previewLimit;
+ const expanded = !compact || (chunk?.id ? expandedChunkId === chunk.id : false);
+ const showingPreview = compact && isLong && !expanded;
+ const visibleContent = showingPreview ? `${displayContent.slice(0, previewLimit).trim()}...` : displayContent;
+
+ return (
+
{loading ? (
) : chunk ? (
-
+
+
+
+ Highlighted source passage
+
{chunk.section_heading && (
{chunk.section_heading}
)}
-
{chunk.content}
+
+ {visibleContent || "No displayable clinical text was available for this indexed passage."}
+
+ {compact && isLong ? (
+
+ setExpandedChunkId((current) => (current === chunk.id ? null : chunk.id))
+ }
+ className={cn(secondaryButton, "mt-3 min-h-9 px-3 text-xs")}
+ data-testid="toggle-full-passage"
+ >
+ {expanded ? "Show passage preview" : "Show full passage"}
+
+ ) : null}
+ {compact ? (
+
+ Full indexed page text remains available in the source text section.
+
+ ) : null}
) : (
@@ -210,12 +415,93 @@ function PinnedSourceEvidence({ loading, chunk }: { loading: boolean; chunk: Chu
);
}
+function IndexedSourceText({
+ text,
+ emptyText,
+ compact = false,
+}: {
+ text: string;
+ emptyText: string;
+ compact?: boolean;
+}) {
+ const blocks = parseIndexedSourceText(text);
+ if (blocks.length === 0) {
+ return
{emptyText}
;
+ }
+
+ return (
+
+ {blocks.map((block) => {
+ if (block.type === "heading") {
+ return block.level === "title" ? (
+
+ {block.text}
+
+ ) : (
+
+ {block.text}
+
+ );
+ }
+
+ if (block.type === "list") {
+ return (
+
+ {block.items.map((item) => (
+
+ {item}
+
+ ))}
+
+ );
+ }
+
+ if (block.type === "table") {
+ return (
+
+ );
+ }
+
+ return (
+
+ {block.text}
+
+ );
+ })}
+
+ );
+}
+
function IndexedTextPanel({
loading,
selectedPage,
chunks,
search,
idPrefix,
+ selectedChunkId,
onSearchChange,
}: {
loading: boolean;
@@ -223,14 +509,20 @@ function IndexedTextPanel({
chunks: ChunkRow[];
search: string;
idPrefix: string;
+ selectedChunkId?: string;
onSearchChange: (value: string) => void;
}) {
const normalizedSearch = search.trim().toLowerCase();
+ const displayChunks = chunks.map((chunk) => ({
+ ...chunk,
+ displayContent: sourceTextForDocumentViewer(chunk.content),
+ }));
const visibleChunks = normalizedSearch
- ? chunks.filter((chunk) =>
- `${chunk.section_heading ?? ""} ${chunk.content}`.toLowerCase().includes(normalizedSearch),
+ ? displayChunks.filter((chunk) =>
+ `${chunk.section_heading ?? ""} ${chunk.displayContent}`.toLowerCase().includes(normalizedSearch),
)
- : chunks.slice(0, 8);
+ : displayChunks.slice(0, 8);
+ const selectedPageText = selectedPage ? sourceTextForIndexedPage(selectedPage.text) : "";
return (
@@ -255,9 +547,10 @@ function IndexedTextPanel({
{loading ? (
) : selectedPage ? (
-
- {selectedPage.text}
-
+
) : (
No extracted text has been indexed for this page yet.
)}
@@ -268,16 +561,34 @@ function IndexedTextPanel({
No indexed passage matched that search.
) : (
visibleChunks.map((chunk) => (
-
+
+ {selectedChunkId === chunk.id ? (
+
+ Highlighted quoted passage
+
+ ) : null}
Page {chunk.page_number ?? "n/a"} · chunk {chunk.chunk_index}
{chunk.section_heading && (
{chunk.section_heading}
)}
-
- {chunk.content}
-
+
))
)}
@@ -288,6 +599,7 @@ function IndexedTextPanel({
}
function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: string; initialPage: number }) {
+ const fullscreenRootRef = useRef(null);
const holderRef = useRef(null);
const canvasRef = useRef(null);
const [pdf, setPdf] = useState(null);
@@ -301,6 +613,8 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri
const [rendering, setRendering] = useState(false);
const [error, setError] = useState(null);
const [loadAttempt, setLoadAttempt] = useState(0);
+ const [isFullscreen, setIsFullscreen] = useState(false);
+ const [fullscreenFallback, setFullscreenFallback] = useState(false);
useEffect(() => {
let active = true;
@@ -356,6 +670,28 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri
};
}, []);
+ useEffect(() => {
+ function updateFullscreenState() {
+ const active = document.fullscreenElement === fullscreenRootRef.current;
+ setIsFullscreen(active);
+ if (active) setFullscreenFallback(false);
+ }
+
+ document.addEventListener("fullscreenchange", updateFullscreenState);
+ return () => document.removeEventListener("fullscreenchange", updateFullscreenState);
+ }, []);
+
+ useEffect(() => {
+ if (!fullscreenFallback) return;
+
+ function exitOnEscape(event: KeyboardEvent) {
+ if (event.key === "Escape") setFullscreenFallback(false);
+ }
+
+ window.addEventListener("keydown", exitOnEscape);
+ return () => window.removeEventListener("keydown", exitOnEscape);
+ }, [fullscreenFallback]);
+
useEffect(() => {
if (!pdf || !canvasRef.current || !holderRef.current) return;
const activePdf = pdf;
@@ -406,10 +742,50 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri
setPageInput(String(bounded));
}
+ async function enterFullscreenFitView() {
+ setFitWidth(true);
+ const element = fullscreenRootRef.current;
+ if (!element) return;
+
+ try {
+ if (document.fullscreenElement === element) {
+ setIsFullscreen(true);
+ return;
+ }
+ if (element.requestFullscreen) {
+ await element.requestFullscreen();
+ setIsFullscreen(true);
+ return;
+ }
+ } catch {
+ // Fall back to a fixed in-app fullscreen surface when native fullscreen is unavailable.
+ }
+
+ setFullscreenFallback(true);
+ setIsFullscreen(true);
+ }
+
+ async function exitFullscreenView() {
+ if (document.fullscreenElement === fullscreenRootRef.current && document.exitFullscreen) {
+ await document.exitFullscreen();
+ }
+ setFullscreenFallback(false);
+ setIsFullscreen(false);
+ }
+
const pagesReady = Boolean(pdf && totalPages > 0 && !loading);
+ const fullscreenActive = isFullscreen || fullscreenFallback;
return (
-
+
setFitWidth(true)}
+ onClick={enterFullscreenFitView}
disabled={!pagesReady}
- aria-label="Fit page width"
+ aria-label="Fit page width and enter fullscreen"
className={cn(
"inline-flex min-h-[44px] min-w-[44px] items-center justify-center gap-2 rounded-lg border px-3 text-xs font-semibold transition",
"disabled:cursor-not-allowed disabled:opacity-45",
- fitWidth
+ fitWidth || fullscreenActive
? "border-[color:var(--primary)]/35 bg-[color:var(--primary-soft)] text-[color:var(--primary)]"
: "border-[color:var(--border)] bg-[color:var(--surface)] text-[color:var(--text)] hover:bg-[color:var(--surface-subtle)]",
)}
@@ -492,6 +868,17 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri
>
+ {fullscreenActive ? (
+
+
+ Exit
+
+ ) : null}
@@ -500,6 +887,7 @@ function PdfCanvasViewer({ url, title, initialPage }: { url: string; title: stri
ref={holderRef}
className={cn(
"polished-scroll relative flex min-h-[46vh] w-full min-w-0 max-w-full justify-center overscroll-contain p-2 [-webkit-overflow-scrolling:touch] sm:min-h-[62vh] sm:p-4",
+ fullscreenActive && "min-h-0 flex-1 sm:min-h-0",
fitWidth
? "overflow-x-hidden overflow-y-auto [touch-action:pan-y]"
: "overflow-auto [touch-action:pan-x_pan-y]",
@@ -564,6 +952,7 @@ export function DocumentViewer({
initialPage: number;
chunkId?: string;
}) {
+ const router = useRouter();
const [document, setDocument] = useState(null);
const [pages, setPages] = useState([]);
const [images, setImages] = useState([]);
@@ -579,14 +968,29 @@ export function DocumentViewer({
const [sourceSearch, setSourceSearch] = useState("");
const [isOnline, setIsOnline] = useState(true);
const [authLoadingTimedOut, setAuthLoadingTimedOut] = useState(false);
+ const [localProjectReady, setLocalProjectReady] = useState(true);
+ const generatedSummaryRef = useRef(null);
const { status: authStatus, isConfigured, authorizationHeader, markSessionExpired } = useAuthSession();
const [serverDemoMode, setServerDemoMode] = useState(process.env.NEXT_PUBLIC_DEMO_MODE === "true" || !isConfigured);
- const clientDemoMode = serverDemoMode;
+ const localNoAuthMode = isLocalNoAuthMode();
+ const clientDemoMode = localNoAuthMode || serverDemoMode;
+ const canUsePrivateApis = localProjectReady && (clientDemoMode || authStatus === "authenticated");
useEffect(() => {
let active = true;
- fetch("/api/setup-status")
- .then((response) => (response.ok ? response.json() : null))
+ readLocalProjectIdentity()
+ .then((identity) => {
+ if (!active) return null;
+ if (!identity?.localServer?.safeLocalOrigin) {
+ setLocalProjectReady(false);
+ setViewerError(unsafeLocalProjectMessage(identity));
+ setLoadingDocument(false);
+ return null;
+ }
+ setLocalProjectReady(true);
+ return fetch("/api/setup-status");
+ })
+ .then((response) => (response?.ok ? response.json() : null))
.then((payload) => {
if (active && typeof payload?.demoMode === "boolean") setServerDemoMode(payload.demoMode);
})
@@ -597,10 +1001,10 @@ export function DocumentViewer({
}, [isConfigured]);
useEffect(() => {
- if (!clientDemoMode && authStatus === "loading") {
+ if (!canUsePrivateApis && authStatus === "loading") {
return () => undefined;
}
- if (!clientDemoMode && authStatus !== "authenticated") {
+ if (!canUsePrivateApis) {
return () => undefined;
}
@@ -614,26 +1018,35 @@ export function DocumentViewer({
}
}, 0);
const detailUrl = `/api/documents/${documentId}${chunkId ? `?chunk=${chunkId}` : ""}`;
- const detailRequest = fetch(detailUrl, {
- signal: controller.signal,
- headers: clientDemoMode ? undefined : authorizationHeader,
- }).then(async (response) => {
- const payload = await response.json();
- if (response.status === 401) markSessionExpired();
- if (!response.ok) throw new Error(payload.error || "Document details could not be loaded.");
- return payload;
- });
- const signedUrlRequest = fetch(`/api/documents/${documentId}/signed-url`, {
- signal: controller.signal,
- headers: clientDemoMode ? undefined : authorizationHeader,
- }).then(async (response) => {
- const payload = await response.json();
- if (response.status === 401) markSessionExpired();
- if (!response.ok) throw new Error(payload.error || "Source preview could not be loaded.");
- return payload;
- });
+ readLocalProjectIdentity()
+ .then((identity) => {
+ if (!identity?.localServer?.safeLocalOrigin) {
+ setLocalProjectReady(false);
+ throw new Error(unsafeLocalProjectMessage(identity));
+ }
+ setLocalProjectReady(true);
- Promise.allSettled([detailRequest, signedUrlRequest])
+ const detailRequest = fetch(detailUrl, {
+ signal: controller.signal,
+ headers: clientDemoMode ? undefined : authorizationHeader,
+ }).then(async (response) => {
+ const payload = await response.json();
+ if (response.status === 401) markSessionExpired();
+ if (!response.ok) throw new Error(payload.error || "Document details could not be loaded.");
+ return payload;
+ });
+ const signedUrlRequest = fetch(`/api/documents/${documentId}/signed-url`, {
+ signal: controller.signal,
+ headers: clientDemoMode ? undefined : authorizationHeader,
+ }).then(async (response) => {
+ const payload = await response.json();
+ if (response.status === 401) markSessionExpired();
+ if (!response.ok) throw new Error(payload.error || "Source preview could not be loaded.");
+ return payload;
+ });
+
+ return Promise.allSettled([detailRequest, signedUrlRequest]);
+ })
.then(([detailResult, signedUrlResult]) => {
if (controller.signal.aborted) return;
@@ -680,6 +1093,7 @@ export function DocumentViewer({
}, [
authStatus,
authorizationHeader,
+ canUsePrivateApis,
clientDemoMode,
documentId,
chunkId,
@@ -700,20 +1114,20 @@ export function DocumentViewer({
}, []);
useEffect(() => {
- if (clientDemoMode || authStatus !== "loading") {
+ if (canUsePrivateApis || authStatus !== "loading") {
return () => undefined;
}
const timeout = window.setTimeout(() => setAuthLoadingTimedOut(true), 3000);
return () => window.clearTimeout(timeout);
- }, [authStatus, clientDemoMode]);
+ }, [authStatus, canUsePrivateApis]);
async function summarize() {
if (!canSummarizeDocument) {
setSummaryError("Load a source document before summarising.");
return;
}
- if (!clientDemoMode && authStatus !== "authenticated") {
+ if (!canUsePrivateApis) {
setSummaryError("Sign in before summarising private documents.");
return;
}
@@ -727,7 +1141,10 @@ export function DocumentViewer({
const payload = await response.json();
if (response.status === 401) markSessionExpired();
if (!response.ok) throw new Error(payload.error || "Summary could not be generated.");
- setSummary(payload);
+ setSummary({ ...payload, answer: cleanClinicalSummaryText(String(payload.answer ?? "")) });
+ window.requestAnimationFrame(() => {
+ generatedSummaryRef.current?.scrollIntoView({ block: "start", behavior: "smooth" });
+ });
} catch (error) {
setSummaryError(error instanceof Error ? error.message : "Summary could not be generated.");
} finally {
@@ -736,15 +1153,14 @@ export function DocumentViewer({
}
const authViewerError =
- !clientDemoMode && authStatus !== "authenticated" && (authStatus !== "loading" || authLoadingTimedOut)
+ !canUsePrivateApis && (authStatus !== "loading" || authLoadingTimedOut)
? isConfigured
? "Sign in to open private source documents."
: "Supabase browser authentication is not configured for private source documents."
: null;
- const effectiveLoadingDocument =
- !clientDemoMode && authStatus !== "authenticated"
- ? authStatus === "loading" && !authLoadingTimedOut && loadingDocument
- : loadingDocument;
+ const effectiveLoadingDocument = !canUsePrivateApis
+ ? authStatus === "loading" && !authLoadingTimedOut && loadingDocument
+ : loadingDocument;
const effectiveViewerError = authViewerError ?? viewerError;
const viewerState = effectiveLoadingDocument
? "loading"
@@ -754,41 +1170,58 @@ export function DocumentViewer({
? "auth-required"
: "error";
const readyDocument = viewerState === "ready" ? document : null;
- const headerTitle =
- readyDocument
- ? readyDocument.title
- : viewerState === "auth-required"
- ? "Sign in required"
- : viewerState === "loading"
- ? "Document"
- : "Source unavailable";
- const headerSubtitle =
- readyDocument
- ? `page ${initialPage} · ${readyDocument.file_name}`
- : viewerState === "loading"
- ? `page ${initialPage} · loading source`
- : (effectiveViewerError ?? "Source unavailable");
- const headerMetadata =
- readyDocument
- ? sourceStatusLabel(normalizeSourceMetadata(readyDocument.metadata))
+ const headerTitle = readyDocument
+ ? readyDocument.title
+ : viewerState === "auth-required"
+ ? "Sign in required"
: viewerState === "loading"
- ? "Loading source metadata"
- : viewerState === "auth-required"
- ? "Private source document"
- : "Source unavailable";
- const canSummarizeDocument =
- viewerState === "ready" && !loadingSummary && (clientDemoMode || authStatus === "authenticated");
+ ? "Document"
+ : "Source unavailable";
+ const headerSubtitle = readyDocument
+ ? `page ${initialPage} · ${readyDocument.file_name}`
+ : viewerState === "loading"
+ ? `page ${initialPage} · loading source`
+ : (effectiveViewerError ?? "Source unavailable");
+ const headerMetadata = readyDocument
+ ? sourceStatusLabel(normalizeSourceMetadata(readyDocument.metadata))
+ : viewerState === "loading"
+ ? "Loading source metadata"
+ : viewerState === "auth-required"
+ ? "Private source document"
+ : "Source unavailable";
+ const canSummarizeDocument = viewerState === "ready" && !loadingSummary && canUsePrivateApis;
const summarizeTitle = canSummarizeDocument
? "Generate a source-backed document summary"
: "Load a source document before summarising";
const selectedPage = pages.find((page) => page.page_number === initialPage) ?? pages[0];
- const selectedChunk = chunks.find((chunk) => chunk.id === chunkId) ?? chunks[0];
+ const selectedChunk = chunkId ? chunks.find((chunk) => chunk.id === chunkId) : undefined;
+ const clinicalImages = images.filter(
+ (image) => image.searchable !== false && (image.clinicalUseClass ?? "clinical_evidence") === "clinical_evidence",
+ );
+ const auditImages = images.filter(
+ (image) =>
+ image.source_kind === "table_crop" &&
+ (image.searchable === false ||
+ ["administrative", "reference"].includes(String(image.clinicalUseClass ?? image.tableRole ?? ""))),
+ );
+ const generatedSummaryText = summary ? cleanClinicalSummaryText(summary.answer) : "";
+ useEffect(() => {
+ if (!chunkId || loadingDocument) return;
+ const target = window.document.querySelector(`[data-source-chunk-id="${CSS.escape(chunkId)}"]`);
+ target?.scrollIntoView({ block: "center", behavior: "smooth" });
+ }, [chunkId, loadingDocument, chunks.length]);
const retryPreview = () => {
setViewerError(null);
setPreviewError(null);
setLoadingDocument(true);
setPreviewAttempt((current) => current + 1);
};
+ const handleDocumentRenamed = (updatedDocument: ClinicalDocument) => {
+ setDocument((current) => (current?.id === updatedDocument.id ? { ...current, ...updatedDocument } : current));
+ };
+ const handleDocumentDeleted = () => {
+ router.push("/");
+ };
return (
@@ -807,9 +1240,7 @@ export function DocumentViewer({
{headerTitle}
-
- {headerSubtitle}
-
+
{headerSubtitle}
{readyDocument ? (
<>
@@ -830,23 +1261,62 @@ export function DocumentViewer({
-
- {loadingSummary ? : }
- Summarise
-
+
+ {readyDocument && (
+
+ )}
+
+ {loadingSummary ? : }
+ Summarise
+
+
+ {(summary || summaryError) && (
+
+ {summary && (
+
+ )}
+ {summaryError && (
+
+ )}
+
+ )}
+
@@ -873,18 +1343,24 @@ export function DocumentViewer({
chunks={chunks}
search={sourceSearch}
idPrefix="mobile-chunk"
+ selectedChunkId={chunkId}
onSearchChange={setSourceSearch}
/>
-
+
{effectiveLoadingDocument ? (
-
+
@@ -944,10 +1421,16 @@ export function DocumentViewer({
-
+
{(items as string[]).slice(0, 5).map((item) => (
- -
+
+ -
+
))}
@@ -1002,37 +1487,37 @@ export function DocumentViewer({
) : null}
-
+
{effectiveLoadingDocument ? (
-
- ) : images.length === 0 ? (
-
No extracted images have been indexed for this document.
+
+ ) : clinicalImages.length === 0 ? (
+
+ No clinically useful tables or diagrams have been indexed for this document.
+
) : (
- images.map((image) =>
)
+ clinicalImages.map((image) =>
)
)}
+ {!effectiveLoadingDocument && auditImages.length > 0 ? (
+
+
+ Administrative/reference tables retained for audit ({auditImages.length})
+
+
+ {auditImages.map((image) => (
+
+ ))}
+
+
+ ) : null}
- {summary && (
-
-
-
- {summary.answer}
-
-
- )}
- {summaryError && (
-
- )}
diff --git a/src/lib/accessible-table-normalization.ts b/src/lib/accessible-table-normalization.ts
new file mode 100644
index 000000000..3064154ba
--- /dev/null
+++ b/src/lib/accessible-table-normalization.ts
@@ -0,0 +1,142 @@
+export type NormalizedAccessibleTable = {
+ header: string[];
+ body: string[][];
+};
+
+function compactCell(value: string | null | undefined) {
+ return String(value ?? "")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function isEmptyCell(value: string | null | undefined) {
+ const normalized = compactCell(value);
+ return !normalized || /^[-–—]+$/.test(normalized);
+}
+
+function isGenericHeader(value: string | null | undefined) {
+ const normalized = compactCell(value);
+ return !normalized || /^column\s+\d+$/i.test(normalized);
+}
+
+function appendCell(existing: string, value: string) {
+ const next = compactCell(value);
+ if (!next) return existing;
+ if (!existing) return next;
+ if (next.length > 16 && existing.toLowerCase().includes(next.toLowerCase())) return existing;
+ return `${existing} ${next}`;
+}
+
+function nearestNamedColumn(index: number, namedIndexes: number[]) {
+ const previous = namedIndexes.filter((namedIndex) => namedIndex < index).at(-1);
+ if (previous !== undefined) return previous;
+ const next = namedIndexes.find((namedIndex) => namedIndex > index);
+ return next ?? index;
+}
+
+function shouldStartNewRow(args: {
+ normalizedRow: string[];
+ rawRow: string[];
+ keptIndexes: number[];
+ previousRows: string[][];
+}) {
+ if (args.previousRows.length === 0) return true;
+ const firstKeptIndex = args.keptIndexes[0] ?? 0;
+ const firstRawCell = compactCell(args.rawRow[firstKeptIndex]);
+ if (firstRawCell) return true;
+
+ const firstNormalizedCell = compactCell(args.normalizedRow[0]);
+ return /^[A-Z](?:\b|$)/.test(firstNormalizedCell) || /^\d{1,2}[.)]\s+/.test(firstNormalizedCell);
+}
+
+function looksLikeHeaderContinuation(args: { rawRow: string[]; keptIndexes: number[] }) {
+ const firstKeptIndex = args.keptIndexes[0] ?? 0;
+ if (compactCell(args.rawRow[firstKeptIndex])) return false;
+ const nonEmptyCount = args.rawRow.filter((cell) => !isEmptyCell(cell)).length;
+ if (nonEmptyCount === 0 || nonEmptyCount > 2) return false;
+ const joined = args.rawRow.map(compactCell).filter(Boolean).join(" ");
+ return !/^[A-F]\b/.test(joined);
+}
+
+export function normalizeAccessibleTable(rows: string[][], columns?: string[] | null): NormalizedAccessibleTable | null {
+ const rawRows = rows
+ .map((row) => row.map(compactCell))
+ .filter((row) => row.some((cell) => !isEmptyCell(cell)));
+ if (!rawRows.length) return null;
+
+ const rawHeader = (columns?.length ? columns : rawRows[0]).map(compactCell);
+ const rawBody = columns?.length ? rawRows : rawRows.slice(1);
+ const sourceColumnCount = Math.max(rawHeader.length, ...rawBody.map((row) => row.length), 1);
+ const paddedHeader = [
+ ...rawHeader,
+ ...Array.from({ length: Math.max(0, sourceColumnCount - rawHeader.length) }, () => ""),
+ ];
+ const namedIndexes = paddedHeader
+ .map((cell, index) => (isGenericHeader(cell) ? null : index))
+ .filter((index): index is number => index !== null);
+
+ const keptIndexes = namedIndexes.length ? namedIndexes : Array.from({ length: sourceColumnCount }, (_, index) => index);
+ const targetBySourceIndex = new Map
();
+
+ for (let sourceIndex = 0; sourceIndex < sourceColumnCount; sourceIndex += 1) {
+ const targetSourceIndex = namedIndexes.length
+ ? isGenericHeader(paddedHeader[sourceIndex])
+ ? nearestNamedColumn(sourceIndex, namedIndexes)
+ : sourceIndex
+ : sourceIndex;
+ const targetIndex = keptIndexes.indexOf(targetSourceIndex);
+ targetBySourceIndex.set(sourceIndex, targetIndex >= 0 ? targetIndex : 0);
+ }
+
+ const header = keptIndexes.map((sourceIndex, index) => {
+ const label = compactCell(paddedHeader[sourceIndex]);
+ return label || (keptIndexes.length === 1 ? "Details" : `Details ${index + 1}`);
+ });
+
+ const bodyRows = [...rawBody];
+ while (bodyRows.length && looksLikeHeaderContinuation({ rawRow: bodyRows[0], keptIndexes })) {
+ const continuation = bodyRows.shift() ?? [];
+ continuation.forEach((cell, sourceIndex) => {
+ if (isEmptyCell(cell)) return;
+ const targetIndex = targetBySourceIndex.get(sourceIndex) ?? sourceIndex;
+ if (targetIndex < 0 || targetIndex >= header.length) return;
+ header[targetIndex] = appendCell(header[targetIndex], cell);
+ });
+ }
+
+ const body: string[][] = [];
+ for (const rawBodyRow of bodyRows) {
+ const paddedRow = [
+ ...rawBodyRow,
+ ...Array.from({ length: Math.max(0, sourceColumnCount - rawBodyRow.length) }, () => ""),
+ ];
+ const normalizedRow = Array.from({ length: header.length }, () => "");
+
+ paddedRow.forEach((cell, sourceIndex) => {
+ if (isEmptyCell(cell)) return;
+ const targetIndex = targetBySourceIndex.get(sourceIndex) ?? sourceIndex;
+ if (targetIndex < 0 || targetIndex >= normalizedRow.length) return;
+ normalizedRow[targetIndex] = appendCell(normalizedRow[targetIndex], cell);
+ });
+
+ if (!normalizedRow.some((cell) => !isEmptyCell(cell))) continue;
+ if (!shouldStartNewRow({ normalizedRow, rawRow: paddedRow, keptIndexes, previousRows: body })) {
+ const previous = body[body.length - 1];
+ normalizedRow.forEach((cell, index) => {
+ previous[index] = appendCell(previous[index] ?? "", cell);
+ });
+ continue;
+ }
+
+ body.push(normalizedRow);
+ }
+
+ const nonEmptyColumnIndexes = header
+ .map((_, index) => index)
+ .filter((index) => body.some((row) => !isEmptyCell(row[index])) || !isGenericHeader(header[index]));
+ const finalHeader = nonEmptyColumnIndexes.map((index) => header[index]);
+ const finalBody = body.map((row) => nonEmptyColumnIndexes.map((index) => row[index] ?? ""));
+
+ if (!finalHeader.length || !finalBody.length) return null;
+ return { header: finalHeader, body: finalBody };
+}
diff --git a/src/lib/answer-formatting.ts b/src/lib/answer-formatting.ts
new file mode 100644
index 000000000..6a0461c8d
--- /dev/null
+++ b/src/lib/answer-formatting.ts
@@ -0,0 +1,240 @@
+export type AnswerDisplayLine = {
+ id: string;
+ label: string | null;
+ text: string;
+ presentation: AnswerLinePresentation;
+};
+
+export type AnswerDisplayTone =
+ | "direct"
+ | "action"
+ | "monitoring"
+ | "medication"
+ | "risk"
+ | "documentation"
+ | "gap"
+ | "comparison"
+ | "source"
+ | "summary";
+
+export type AnswerLinePresentation = {
+ tone: AnswerDisplayTone;
+ symbol: string;
+ label: string;
+};
+
+export type AnswerDisplayMode = "direct" | "checklist" | "clinical_pathway" | "comparison" | "summary" | "evidence_gap";
+
+export type ParsedAnswerDisplay =
+ | { type: "paragraph"; lines: AnswerDisplayLine[]; mode: AnswerDisplayMode }
+ | { type: "bullets"; lines: AnswerDisplayLine[]; mode: AnswerDisplayMode };
+
+const knownAnswerLabels = new Set([
+ "bottom line",
+ "required actions",
+ "monitoring",
+ "monitoring/timing",
+ "medication/dose details",
+ "dose detail",
+ "medication point",
+ "table evidence",
+ "threshold/action",
+ "risk/escalation",
+ "escalation/risk",
+ "workflow step",
+ "documentation/forms",
+ "source gaps",
+ "section summary",
+ "source point",
+]);
+
+function normalizeInline(value: string) {
+ return value.replace(/[ \t]+/g, " ").trim();
+}
+
+function stripBulletPrefix(value: string) {
+ return value.replace(/^(?:[-*•]|\d+[.)])\s+/, "").trim();
+}
+
+function splitInlineBullets(value: string) {
+ const trimmed = value.trim();
+ if (!trimmed) return [];
+
+ const newlineParts = trimmed
+ .split(/\r?\n+/)
+ .map((line) => line.trim())
+ .filter(Boolean);
+ if (newlineParts.filter((line) => /^(?:[-*•]|\d+[.)])\s+/.test(line)).length >= 2) {
+ return newlineParts.map(stripBulletPrefix);
+ }
+
+ const inlineParts = trimmed
+ .split(/(?:^|\s)-\s+(?=(?:\*\*)?[A-Z0-9])/)
+ .map((part) => part.trim())
+ .filter(Boolean);
+ if (inlineParts.length >= 2) return inlineParts;
+
+ return [trimmed];
+}
+
+function splitLabel(value: string): Pick {
+ const match = value.match(/^([^:]{2,44}):\s+(.+)$/);
+ if (!match) return { label: null, text: value };
+
+ const rawLabel = normalizeInline(match[1].replace(/\*\*/g, ""));
+ const labelKey = rawLabel.toLowerCase();
+ if (!knownAnswerLabels.has(labelKey) && !/^[A-Z][A-Za-z/ -]{2,40}$/.test(rawLabel)) {
+ return { label: null, text: value };
+ }
+
+ return {
+ label: rawLabel,
+ text: normalizeInline(match[2]),
+ };
+}
+
+function inferPresentation(label: string | null, text: string): AnswerLinePresentation {
+ const labelKey = (label ?? "").toLowerCase();
+ if (/\bbottom line\b/.test(labelKey)) {
+ return { tone: "direct", symbol: "✓", label: label ?? "Bottom line" };
+ }
+ if (/\b(?:source gaps?|gap|unsupported)\b/.test(labelKey)) {
+ return { tone: "gap", symbol: "?", label: label ?? "Source gap" };
+ }
+ if (/\b(?:risk|escalation|red flag|safety)\b/.test(labelKey)) {
+ return { tone: "risk", symbol: "!", label: label ?? "Risk" };
+ }
+ if (/\b(?:medication|dose|titration|prescrib)\b/.test(labelKey)) {
+ return { tone: "medication", symbol: "Rx", label: label ?? "Medication" };
+ }
+ if (/\b(?:monitoring|timing|threshold|table evidence)\b/.test(labelKey)) {
+ return { tone: "monitoring", symbol: "⏱", label: label ?? "Monitoring" };
+ }
+ if (/\b(?:required actions?|workflow|action)\b/.test(labelKey)) {
+ return { tone: "action", symbol: "→", label: label ?? "Action" };
+ }
+ if (/\b(?:documentation|forms?|record|audit)\b/.test(labelKey)) {
+ return { tone: "documentation", symbol: "§", label: label ?? "Document" };
+ }
+ if (/\b(?:compare|comparison|conflict)\b/.test(labelKey)) {
+ return { tone: "comparison", symbol: "↔", label: label ?? "Compare" };
+ }
+ if (/\b(?:source|citation|quote|evidence)\b/.test(labelKey)) {
+ return { tone: "source", symbol: "#", label: label ?? "Source" };
+ }
+ if (/\b(?:summary|overview|section summary)\b/.test(labelKey)) {
+ return { tone: "summary", symbol: "•", label: label ?? "Summary" };
+ }
+
+ const combined = text.toLowerCase();
+ if (/\b(?:gap|insufficient|unsupported|not contain|not enough|unclear|missing)\b/.test(combined)) {
+ return { tone: "gap", symbol: "?", label: label ?? "Source gap" };
+ }
+ if (/\b(?:risk|escalat|urgent|immediate|red flag|cease|withhold|stop|contraindicat|avoid|emergency)\b/.test(combined)) {
+ return { tone: "risk", symbol: "!", label: label ?? "Risk" };
+ }
+ if (/\b(?:monitor|timing|weekly|monthly|hours?|days?|weeks?|anc|fbc|wbc|blood test|threshold|level)\b/.test(combined)) {
+ return { tone: "monitoring", symbol: "⏱", label: label ?? "Monitoring" };
+ }
+ if (/\b(?:dose|mg|mcg|route|oral|intramuscular|im\b|po\b|clozapine|lithium|lorazepam|haloperidol|olanzapine|medication)\b/.test(combined)) {
+ return { tone: "medication", symbol: "Rx", label: label ?? "Medication" };
+ }
+ if (/\b(?:document|form|record|audit|consent|register|file|note)\b/.test(combined)) {
+ return { tone: "documentation", symbol: "§", label: label ?? "Document" };
+ }
+ if (/\b(?:compare|versus|difference|whereas|while|document-specific|conflict)\b/.test(combined)) {
+ return { tone: "comparison", symbol: "↔", label: label ?? "Compare" };
+ }
+ if (/\b(?:source|citation|quote|evidence|excerpt)\b/.test(combined)) {
+ return { tone: "source", symbol: "#", label: label ?? "Source" };
+ }
+ if (/\b(?:action|required|must|should|complete|refer|review|arrange|contact|notify|assess)\b/.test(combined)) {
+ return { tone: "action", symbol: "→", label: label ?? "Action" };
+ }
+ if (/\b(?:summary|overview|bottom line|key point)\b/.test(combined)) {
+ return { tone: "summary", symbol: "•", label: label ?? "Summary" };
+ }
+ return { tone: "direct", symbol: "✓", label: label ?? "Point" };
+}
+
+function inferDisplayMode(lines: AnswerDisplayLine[]): AnswerDisplayMode {
+ const presentations = lines.map((line) => line.presentation);
+ if (presentations.some((item) => item.tone === "gap")) return "evidence_gap";
+ if (presentations.some((item) => item.tone === "comparison")) return "comparison";
+ if (presentations.some((item) => item.tone === "risk" || item.tone === "medication" || item.tone === "monitoring")) {
+ return "clinical_pathway";
+ }
+ if (lines.length >= 2 && presentations.some((item) => item.tone === "action" || item.tone === "documentation")) {
+ return "checklist";
+ }
+ if (lines.length >= 3) return "summary";
+ return "direct";
+}
+
+export function answerLinePresentation(line: AnswerDisplayLine): AnswerLinePresentation {
+ return line.presentation ?? inferPresentation(line.label, line.text);
+}
+
+function buildAnswerLine(part: string, index: number, prefix: string): AnswerDisplayLine {
+ const parsed = splitLabel(part);
+ return {
+ id: `${prefix}:${index}:${part.slice(0, 48)}`,
+ ...parsed,
+ presentation: inferPresentation(parsed.label, parsed.text),
+ };
+}
+
+export function parseAnswerDisplayContent(value: string): ParsedAnswerDisplay {
+ const cleaned = value
+ .replace(/\r/g, "\n")
+ .replace(/[ \t]+/g, " ")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+ if (!cleaned) {
+ return {
+ type: "paragraph",
+ mode: "evidence_gap",
+ lines: [
+ {
+ id: "empty",
+ label: null,
+ text: "No usable answer text for this result.",
+ presentation: inferPresentation(null, "No usable answer text for this result."),
+ },
+ ],
+ };
+ }
+
+ const parts = splitInlineBullets(cleaned)
+ .map((part) => normalizeInline(part))
+ .filter((part) => part.length > 0);
+
+ if (parts.length >= 2) {
+ const lines = parts.map((part, index) => buildAnswerLine(part, index, "bullet"));
+ return {
+ type: "bullets",
+ mode: inferDisplayMode(lines),
+ lines,
+ };
+ }
+
+ const semicolonParts = cleaned
+ .split(/\s*;\s*/)
+ .map((part) => normalizeInline(part))
+ .filter((part) => part.length > 18);
+ if (semicolonParts.length >= 3) {
+ const lines = semicolonParts.map((part, index) => buildAnswerLine(part, index, "semicolon"));
+ return {
+ type: "bullets",
+ mode: inferDisplayMode(lines),
+ lines,
+ };
+ }
+
+ const lines = [buildAnswerLine(normalizeInline(cleaned), 0, "paragraph")];
+ return {
+ type: "paragraph",
+ mode: inferDisplayMode(lines),
+ lines,
+ };
+}
diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts
new file mode 100644
index 000000000..528d0f410
--- /dev/null
+++ b/src/lib/answer-ranking.ts
@@ -0,0 +1,304 @@
+import { classifyRagQuery, hasDoseEvidenceSupport, normalizedClinicalSearchTokens } from "@/lib/clinical-search";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
+import { sourceTextForModel } from "@/lib/source-text-sanitizer";
+import type { RagAnswer, RagQueryClass, SearchResult } from "@/lib/types";
+
+const answerRankStrategy = "query_focused_answer_evidence_v1";
+
+const answerBoilerplatePattern =
+ /\b(?:uncontrolled when printed|document control|review date|version\s+\d|page\s+\d+\s+of\s+\d+|copyright|confidential)\b/i;
+
+const queryTermExclusions = new Set([
+ "recommended",
+ "recommend",
+ "summary",
+ "summarize",
+ "summarise",
+ "overview",
+ "information",
+ "guidance",
+ "approach",
+ "therapy",
+]);
+
+const fixedHighYieldPatterns = [
+ /\b(?:clozapine|lithium|ECT|FBC|ANC|myocarditis|neutropenia|metabolic|constipation|ECG)\b/gi,
+ /\b(?:withhold|withholding|cease|ceased|stop|stopping|discontinue\w*|contraindicat\w*|avoid|urgent|escalat\w*|red flag\w*)\b/gi,
+ /\b\d+(?:\.\d+)?\s?(?:mg|mcg|g|mmol\/L|days?|weeks?|months?|hours?|minutes?|%)\b/gi,
+ /\brating\s+\d+(?:\s*[-–]\s*\d+)?\b/gi,
+];
+
+function clampScore(value: number) {
+ if (!Number.isFinite(value)) return 0;
+ return Math.max(0, Math.min(1.5, value));
+}
+
+function normalizeText(value: string) {
+ return value
+ .replace(/([a-z])([A-Z])/g, "$1 $2")
+ .toLowerCase()
+ .replace(/[^a-z0-9%/.]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function textTokens(value: string) {
+ return normalizeText(value)
+ .split(/\s+/)
+ .map((token) => (token.endsWith("s") && !token.endsWith("ss") ? token.slice(0, -1) : token))
+ .filter((token) => token.length > 2);
+}
+
+function imageEvidenceText(result: SearchResult) {
+ return (result.images ?? [])
+ .filter((image) => isClinicalImageEvidence(image))
+ .map((image) =>
+ [
+ image.image_type,
+ image.sourceKind ?? image.source_kind,
+ image.tableLabel,
+ image.tableTitle,
+ image.tableTextSnippet,
+ image.caption,
+ ...(image.labels ?? []),
+ ]
+ .filter(Boolean)
+ .join(" "),
+ )
+ .join(" ");
+}
+
+function resultTexts(result: SearchResult) {
+ const labels = (result.document_labels ?? []).map((label) => `${label.label} ${label.label_type}`).join(" ");
+ const metadataText = `${labels} ${result.document_summary ?? ""}`;
+ const titleText = `${result.title} ${result.file_name}`;
+ const sectionText = result.section_heading ?? "";
+ const contentText = `${sourceTextForModel(result.content)} ${imageEvidenceText(result)}`;
+ return {
+ title: normalizeText(titleText),
+ section: normalizeText(sectionText),
+ content: normalizeText(contentText),
+ metadata: normalizeText(metadataText),
+ adjacent: normalizeText(result.adjacent_context ?? ""),
+ combined: normalizeText(`${titleText} ${sectionText} ${contentText} ${metadataText} ${result.adjacent_context ?? ""}`),
+ };
+}
+
+function uniqueQueryTokens(query: string) {
+ const tokens = normalizedClinicalSearchTokens(query);
+ const fallback = tokens.length ? tokens : textTokens(query);
+ return Array.from(new Set(fallback.filter((token) => token.length > 2)));
+}
+
+function tokenCoverage(tokens: string[], haystack: string) {
+ if (!tokens.length || !haystack) return 0;
+ const hits = tokens.filter((token) => haystack.includes(token));
+ return hits.length / tokens.length;
+}
+
+function phraseScore(query: string, haystack: string) {
+ const queryTokens = uniqueQueryTokens(query);
+ if (queryTokens.length < 2 || !haystack) return 0;
+ const phrases: string[] = [];
+ for (let size = Math.min(4, queryTokens.length); size >= 2; size -= 1) {
+ for (let index = 0; index <= queryTokens.length - size; index += 1) {
+ phrases.push(queryTokens.slice(index, index + size).join(" "));
+ }
+ }
+ const hits = phrases.filter((phrase) => haystack.includes(phrase)).length;
+ return Math.min(1, hits / Math.max(1, Math.min(phrases.length, 4)));
+}
+
+function classSignalScore(queryClass: RagQueryClass, result: SearchResult, combinedText: string) {
+ const hasTableEvidence = (result.images ?? []).some(
+ (image) =>
+ isClinicalImageEvidence(image) &&
+ (image.image_type === "clinical_table" ||
+ image.image_type === "medication_chart" ||
+ image.sourceKind === "table_crop" ||
+ image.source_kind === "table_crop"),
+ );
+
+ if (queryClass === "table_threshold") {
+ return (
+ (/\b(?:threshold|cut\s?off|withhold|cease|stop|anc|fbc|level|range|criteria|rating|table|chart)\b/i.test(
+ combinedText,
+ )
+ ? 0.13
+ : 0) + (hasTableEvidence ? 0.08 : 0)
+ );
+ }
+ if (queryClass === "medication_dose_risk") {
+ return hasDoseEvidenceSupport(result)
+ ? 0.2
+ : /\b(?:risk|toxicity|side effect|monitor)\b/i.test(combinedText)
+ ? 0.06
+ : 0;
+ }
+ if (queryClass === "document_lookup") {
+ return /\b(?:document|guideline|procedure|protocol|form|section|appendix)\b/i.test(combinedText) ? 0.08 : 0;
+ }
+ if (queryClass === "comparison") {
+ return /\b(?:requirement|process|procedure|criteria|include|document|action)\b/i.test(combinedText) ? 0.04 : 0;
+ }
+ if (queryClass === "broad_summary") {
+ return /\b(?:summary|overview|purpose|scope|procedure|requirement|action)\b/i.test(combinedText) ? 0.04 : 0;
+ }
+ return 0;
+}
+
+function sourceQualityScore(result: SearchResult) {
+ const metadata = result.source_metadata;
+ let score = 0;
+ if (metadata?.document_status === "current") score += 0.035;
+ if (metadata?.document_status === "outdated") score -= 0.08;
+ if (metadata?.clinical_validation_status === "approved") score += 0.035;
+ if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.02;
+ if (metadata?.extraction_quality === "good") score += 0.035;
+ if (metadata?.extraction_quality === "poor") score -= 0.06;
+ return score;
+}
+
+function answerEvidenceScore(query: string, result: SearchResult, queryClass: RagQueryClass) {
+ const texts = resultTexts(result);
+ const tokens = uniqueQueryTokens(query);
+ const base = Math.min(1, result.hybrid_score ?? result.similarity ?? 0) * 0.28;
+ const contentCoverage = tokenCoverage(tokens, texts.content);
+ const titleCoverage = tokenCoverage(tokens, texts.title);
+ const sectionCoverage = tokenCoverage(tokens, texts.section);
+ const metadataCoverage = tokenCoverage(tokens, texts.metadata);
+ const combinedCoverage = tokenCoverage(tokens, texts.combined);
+ const adjacentCoverage = tokenCoverage(tokens, texts.adjacent);
+ const phraseCoverage = phraseScore(query, texts.combined);
+ const directnessScore =
+ contentCoverage * 0.34 + titleCoverage * 0.12 + sectionCoverage * 0.1 + metadataCoverage * 0.08;
+ const weakOverlapPenalty = combinedCoverage < 0.2 ? -0.18 : combinedCoverage < 0.34 ? -0.07 : 0;
+ const adjacentOnlyPenalty = contentCoverage < 0.16 && adjacentCoverage > contentCoverage ? -0.08 : 0;
+ const boilerplatePenalty = answerBoilerplatePattern.test(result.content) && contentCoverage < 0.35 ? -0.08 : 0;
+ const coreConceptTokens = tokens.filter(
+ (token) =>
+ ![
+ "dose",
+ "dosing",
+ "dosage",
+ "medication",
+ "medicine",
+ "route",
+ "oral",
+ "intramuscular",
+ "monitor",
+ "monitoring",
+ "risk",
+ ].includes(token),
+ );
+ const missingCoreConceptPenalty =
+ queryClass === "medication_dose_risk" &&
+ coreConceptTokens.length > 0 &&
+ !coreConceptTokens.some((token) => texts.combined.includes(token))
+ ? -0.22
+ : 0;
+ const titleOnlyDosePenalty =
+ queryClass === "medication_dose_risk" &&
+ titleCoverage >= 0.4 &&
+ !hasDoseEvidenceSupport(result)
+ ? -0.18
+ : 0;
+
+ return clampScore(
+ base +
+ directnessScore +
+ phraseCoverage * 0.12 +
+ classSignalScore(queryClass, result, texts.combined) +
+ sourceQualityScore(result) +
+ weakOverlapPenalty +
+ adjacentOnlyPenalty +
+ boilerplatePenalty +
+ titleOnlyDosePenalty +
+ missingCoreConceptPenalty,
+ );
+}
+
+export function rankAnswerEvidence(
+ query: string,
+ results: SearchResult[],
+ queryClass: RagQueryClass = classifyRagQuery(query).queryClass,
+) {
+ const ranked = results
+ .map((result, index) => ({
+ result,
+ index,
+ score: answerEvidenceScore(query, result, queryClass),
+ }))
+ .sort(
+ (a, b) =>
+ b.score - a.score ||
+ (b.result.hybrid_score ?? b.result.similarity ?? 0) - (a.result.hybrid_score ?? a.result.similarity ?? 0) ||
+ a.index - b.index,
+ );
+
+ return {
+ rankedResults: ranked.map((item) => item.result),
+ scoresByChunkId: new Map(ranked.map((item) => [item.result.id, Number(item.score.toFixed(4))])),
+ topScore: ranked[0] ? Number(ranked[0].score.toFixed(4)) : 0,
+ rankedSourceCount: ranked.length,
+ strategy: answerRankStrategy,
+ queryClass,
+ };
+}
+
+function escapeRegExp(value: string) {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function queryHighlightPatterns(query?: string) {
+ if (!query) return [];
+ const tokens = uniqueQueryTokens(query).filter((token) => token.length >= 4 && !queryTermExclusions.has(token));
+ const patterns: RegExp[] = [];
+ const normalizedQuery = normalizeText(query);
+ const queryPhrase = tokens.length >= 2 ? tokens.join(" ") : "";
+ if (queryPhrase && normalizedQuery.includes(queryPhrase)) {
+ patterns.push(new RegExp(`\\b${escapeRegExp(queryPhrase).replace(/\\ /g, "\\s+")}\\b`, "gi"));
+ }
+ for (const token of tokens.slice(0, 6).sort((a, b) => b.length - a.length)) {
+ patterns.push(new RegExp(`\\b${escapeRegExp(token)}\\w*\\b`, "gi"));
+ }
+ return patterns;
+}
+
+function applyBoldPatternOutsideExisting(text: string, pattern: RegExp, maxMatches: number) {
+ let applied = 0;
+ const segments = text.split(/(\*\*[^*]+\*\*)/g);
+ return segments
+ .map((segment) => {
+ if (segment.startsWith("**") && segment.endsWith("**")) return segment;
+ return segment.replace(pattern, (match) => {
+ if (applied >= maxMatches) return match;
+ if (!/[A-Za-z0-9]/.test(match)) return match;
+ applied += 1;
+ return `**${match}**`;
+ });
+ })
+ .join("");
+}
+
+export function boldHighYieldClinicalText(text: string, query?: string) {
+ if (!text.trim()) return text;
+ if (query === undefined) return text;
+ if (/[{}\[\]]/.test(text) && /"?(?:answer|heading|citation_chunk_ids|chunk_id)"?\s*:/i.test(text)) return text;
+ let output = text;
+ for (const pattern of [...queryHighlightPatterns(query), ...fixedHighYieldPatterns]) {
+ output = applyBoldPatternOutsideExisting(output, pattern, 8);
+ }
+ return output;
+}
+
+export function boldRagAnswerHighYieldText(answer: RagAnswer, query: string): RagAnswer {
+ return {
+ ...answer,
+ answer: boldHighYieldClinicalText(answer.answer, query),
+ answerSections: answer.answerSections?.map((section) => ({
+ ...section,
+ body: boldHighYieldClinicalText(section.body, query),
+ })),
+ };
+}
diff --git a/src/lib/bulk-import.ts b/src/lib/bulk-import.ts
index fd53e62ea..d18123313 100644
--- a/src/lib/bulk-import.ts
+++ b/src/lib/bulk-import.ts
@@ -2,10 +2,12 @@ import { createHash, randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import path from "node:path";
+import { smartDocumentTitle } from "@/lib/document-naming";
export type ImportCliArgs = {
path: string;
ownerEmail?: string;
+ ownerId?: string;
batchName?: string;
include: string;
limit?: number;
@@ -59,6 +61,7 @@ export function parseImportCliArgs(argv: string[]): ImportCliArgs {
return {
path: args.path,
ownerEmail: typeof args.ownerEmail === "string" ? args.ownerEmail : undefined,
+ ownerId: typeof args.ownerId === "string" ? args.ownerId : undefined,
batchName: typeof args.batchName === "string" ? args.batchName : undefined,
include: typeof args.include === "string" ? args.include : "**/*.pdf",
limit: typeof args.limit === "string" ? Number.parseInt(args.limit, 10) : undefined,
@@ -73,7 +76,7 @@ export function safeFileName(fileName: string) {
}
export function titleFromFileName(fileName: string) {
- return fileName.replace(/\.[^.]+$/, "");
+ return smartDocumentTitle(fileName);
}
export function buildImportStoragePath(ownerId: string, documentId: string, fileName: string) {
@@ -85,9 +88,7 @@ export function formatExactDuplicateSkip(
duplicate: ExistingImportDocument,
options: { dryRun?: boolean } = {},
) {
- const prefix = options.dryRun
- ? "DRY RUN DUPLICATE exact copy would be skipped"
- : "DUPLICATE exact copy skipped";
+ const prefix = options.dryRun ? "DRY RUN DUPLICATE exact copy would be skipped" : "DUPLICATE exact copy skipped";
const matchedSource = duplicate.source_path || duplicate.storage_path;
return `${prefix} ${file.relativePath} (matches "${duplicate.title}" at ${matchedSource})`;
}
@@ -116,7 +117,11 @@ export async function hashFile(filePath: string) {
return hash.digest("hex");
}
-export async function scanImportFiles(root: string, include = "**/*.pdf", limit?: number): Promise {
+export async function scanImportFiles(
+ root: string,
+ include = "**/*.pdf",
+ limit?: number,
+): Promise {
const rootStat = await stat(root);
if (!rootStat.isDirectory()) {
throw new Error(`Import path is not a directory: ${root}`);
diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts
index 81a088ff8..488887c89 100644
--- a/src/lib/chunking.ts
+++ b/src/lib/chunking.ts
@@ -3,6 +3,15 @@ import type { ChunkInput, DocumentChunk } from "@/lib/types";
const sentenceBoundary = /(?<=[.!?])\s+/;
const paragraphBoundary = /\n{2,}/;
+const metadataNoisePatterns: RegExp[] = [
+ /\b(?:copyright|all rights reserved|document revision|do not distribute|downloaded from|www\.\S+)\b/i,
+ /\b(?:version|revision)\s+\d+\s*$/i,
+];
+const lineNoisePatterns: RegExp[] = [
+ /\b(page|p\.?)\s*\d+\s*(?:\/\s*\d+)?\b/i,
+ /^\s*[-*_]{3,}\s*$/,
+ /^\s*[\u25cf\u25e6\u2022]\s*$/,
+];
export function estimateTokens(text: string) {
return Math.ceil(text.length / 4);
@@ -22,15 +31,83 @@ export function detectHeading(text: string) {
return null;
}
+function normalizeLookupText(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .replace(/\b(?:table|figure|appendix)\b/g, "")
+ .trim();
+}
+
+function looksLikeMetadataNoise(line: string) {
+ if (!line || line.length <= 2) return true;
+ if (/^\d+$/.test(line)) return true;
+ if (metadataNoisePatterns.some((pattern) => pattern.test(line))) return true;
+ if (lineNoisePatterns.some((pattern) => pattern.test(line))) return true;
+ return false;
+}
+
+function removePageNoise(text: string) {
+ return text
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line === "" || !looksLikeMetadataNoise(line))
+ .join("\n")
+ .replace(/[ \t]+\n/g, "\n")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+}
+
+function extractSectionHeadings(text: string) {
+ return text
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => {
+ if (!line || line.length < 4) return false;
+ if (looksLikeMetadataNoise(line)) return false;
+ if (line.length < 90 && /^[A-Z][A-Za-z0-9\s,;:()\/\-\[\]]+$/.test(line)) return true;
+ if (/^\d+\.?\s+[A-Z]/.test(line) && line.length < 110) return true;
+ return false;
+ });
+}
+
+function sectionAnchorId(heading: string | null) {
+ if (!heading) return null;
+ return (
+ heading
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, "-")
+ .replace(/(^-|-$)/g, "")
+ .slice(0, 40) || null
+ );
+}
+
+function imageMatchScore(lookupText: string, sourceText: string) {
+ if (!lookupText || !sourceText) return 0;
+ const source = new Set(sourceText.split(/\s+/).filter(Boolean));
+ const hits = lookupText
+ .split(/\s+/)
+ .filter(Boolean)
+ .filter((token) => source.has(token)).length;
+ return hits;
+}
+
+function dedupeChunkFingerprint(text: string) {
+ return normalizeLookupText(text).replace(/\s+/g, " ").trim();
+}
+
export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, overlap = env.CHUNK_OVERLAP) {
- const clean = text
+ const clean = removePageNoise(text)
.replace(/[ \t]+\n/g, "\n")
.replace(/[ \t]+/g, " ")
.trim();
if (!clean) return [];
if (clean.length <= chunkSize) return [clean];
- const paragraphs = clean.split(paragraphBoundary).map((paragraph) => paragraph.trim()).filter(Boolean);
+ const paragraphs = clean
+ .split(paragraphBoundary)
+ .map((paragraph) => paragraph.trim())
+ .filter(Boolean);
if (paragraphs.length > 1) {
const chunks: string[] = [];
let current = "";
@@ -61,6 +138,20 @@ export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, o
return chunkTextBySentence(clean, chunkSize, overlap);
}
+function readableOverlapStart(clean: string, end: number, overlap: number) {
+ if (overlap <= 0) return end;
+ let start = Math.max(0, end - overlap);
+
+ while (start > 0 && start < end && /\S/.test(clean[start - 1] ?? "") && /\S/.test(clean[start] ?? "")) {
+ start -= 1;
+ }
+ while (start < end && /\s/.test(clean[start] ?? "")) {
+ start += 1;
+ }
+
+ return start < end ? start : Math.max(0, end - overlap);
+}
+
function chunkTextBySentence(clean: string, chunkSize: number, overlap: number) {
const chunks: string[] = [];
let start = 0;
@@ -80,30 +171,91 @@ function chunkTextBySentence(clean: string, chunkSize: number, overlap: number)
const chunk = clean.slice(start, end).trim();
if (chunk) chunks.push(chunk);
if (end >= clean.length) break;
- start = Math.max(0, end - overlap);
+ start = readableOverlapStart(clean, end, overlap);
}
return chunks;
}
-export function buildImageTag(image: { id: string; caption: string }) {
- return `[[IMAGE_DATA_START]] Image ID: ${image.id}; Description: ${image.caption} [[IMAGE_DATA_END]]`;
+function compactImageText(value: string | null | undefined, limit = 900) {
+ const text = String(value ?? "")
+ .replace(/\s+/g, " ")
+ .trim();
+ if (!text) return "";
+ return text.length > limit ? `${text.slice(0, limit - 3).trim()}...` : text;
+}
+
+export function buildImageTag(image: {
+ id: string;
+ caption: string;
+ imageType?: string | null;
+ sourceKind?: string | null;
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableTextSnippet?: string | null;
+}) {
+ const parts = [
+ `Image ID: ${image.id}`,
+ image.sourceKind ? `Source kind: ${image.sourceKind}` : "",
+ image.imageType ? `Image type: ${image.imageType}` : "",
+ image.tableLabel ? `Table label: ${image.tableLabel}` : "",
+ image.tableTitle ? `Table title: ${image.tableTitle}` : "",
+ image.tableRole ? `Table role: ${image.tableRole}` : "",
+ image.tableTextSnippet ? `Table text: ${compactImageText(image.tableTextSnippet)}` : "",
+ `Description: ${image.caption}`,
+ ].filter(Boolean);
+ return `[[IMAGE_DATA_START]] ${parts.join("; ")} [[IMAGE_DATA_END]]`;
}
export function buildChunks(inputs: ChunkInput[]) {
const chunks: DocumentChunk[] = [];
+ const chunkFingerprint = new Map();
for (const input of inputs) {
const pageImages = input.images ?? [];
const imageContext = pageImages.map(buildImageTag).join("\n");
- const pageText = [input.pageText, imageContext].filter(Boolean).join("\n\n");
+ const sectionPath = extractSectionHeadings(input.pageText);
+ const pageText = [removePageNoise(input.pageText), imageContext].filter(Boolean).join("\n\n");
+ const pageLookupText = normalizeLookupText(input.pageText);
const pageChunks = chunkTextWithOverlap(pageText);
pageChunks.forEach((content, pageChunkIndex) => {
+ const contentLookup = normalizeLookupText(content);
+ const heading = detectHeading(content);
+ const sectionContext = sectionPath.includes(heading ?? "") ? sectionPath : [...sectionPath];
+ const sectionAnchor = sectionAnchorId(heading);
const referencedImageIds = pageImages
- .filter((image) => content.includes(image.id) || content.includes(image.caption))
+ .filter((image) => {
+ const label = normalizeLookupText(image.tableLabel ?? "");
+ const title = normalizeLookupText(image.tableTitle ?? "");
+ const caption = normalizeLookupText(image.caption);
+ const imageText = [label, title, caption]
+ .filter(Boolean)
+ .flatMap((value) => value.split(/\s+/).filter(Boolean));
+ const imageLookup = imageText.join(" ");
+ const headerBoost =
+ heading && imageText.some((token) => normalizeLookupText(heading).includes(token)) ? 1 : 0;
+ const direct =
+ imageMatchScore(caption, contentLookup) >= 1 || imageMatchScore(imageLookup, contentLookup) >= 2;
+ const pathHit =
+ sectionContext.some((candidate) =>
+ normalizeLookupText(candidate)
+ .split(/\s+/)
+ .some((token) => imageLookup.includes(token)),
+ ) && image.sourceKind !== "embedded";
+ return direct || pathHit || (image.sourceKind === "table_crop" && headerBoost > 0) || headerBoost >= 1;
+ })
.map((image) => image.id);
+ const fingerprint = dedupeChunkFingerprint(content);
+ if (fingerprint && chunkFingerprint.has(fingerprint)) {
+ return;
+ }
+
+ if (fingerprint) {
+ chunkFingerprint.set(fingerprint, chunks.length);
+ }
chunks.push({
document_id: input.documentId,
page_number: input.pageNumber,
@@ -117,6 +269,9 @@ export function buildChunks(inputs: ChunkInput[]) {
page_chunk_index: pageChunkIndex,
page_start: input.pageNumber,
page_end: input.pageNumber,
+ heading_lookup: pageLookupText,
+ subsection_path: sectionContext,
+ section_anchor: sectionAnchor,
},
});
});
diff --git a/src/lib/clinical-safety.ts b/src/lib/clinical-safety.ts
index 900b6f0cf..ca909ac2c 100644
--- a/src/lib/clinical-safety.ts
+++ b/src/lib/clinical-safety.ts
@@ -1,4 +1,6 @@
import { documentCitationHref, formatCitationLabel } from "@/lib/citations";
+import { queryCoreTerms } from "@/lib/evidence-relevance";
+import { sourceTextForDisplay } from "@/lib/source-text-sanitizer";
import type { Citation, RagAnswer, SearchResult } from "@/lib/types";
export type SafetyFindingKind =
@@ -62,7 +64,7 @@ function normalizeText(text: string) {
}
function conciseSourceText(text: string) {
- const normalized = normalizeText(text);
+ const normalized = normalizeText(sourceTextForDisplay(text));
if (normalized.length <= 260) return normalized;
return `${normalized.slice(0, 257).trim()}...`;
}
@@ -80,19 +82,38 @@ function citationFromSource(source: SearchResult): Citation {
};
}
+function hasQueryConceptOverlap(text: string, terms: string[]) {
+ if (terms.length === 0) return true;
+ const haystack = text.toLowerCase();
+ return terms.some((term) => haystack.includes(term.toLowerCase()));
+}
+
export function extractSafetyFindings(answer: RagAnswer | null | undefined, limit = 5): SafetyFinding[] {
if (!answer?.grounded) return [];
+ if (answer.relevance && !answer.relevance.isSourceBacked) return [];
+
+ const sourceByChunkId = new Map((answer.sources ?? []).map((source) => [source.id, source]));
+ const queryTerms = queryCoreTerms(answer.smartPanel?.query ?? "");
+ const relevanceTerms = answer.relevance?.matchedTerms ?? [];
+ const coreTerms = queryTerms.length ? queryTerms : relevanceTerms;
const candidates = [
- ...(answer.quoteCards ?? []).map((quote) => ({
- id: quote.chunk_id,
- text: quote.quote,
- citation: quote,
- })),
+ ...(answer.quoteCards ?? []).map((quote) => {
+ const source = sourceByChunkId.get(quote.chunk_id);
+ return {
+ id: quote.chunk_id,
+ text: quote.quote,
+ citation: quote,
+ source,
+ sourceStrength: quote.source_strength ?? source?.source_strength,
+ };
+ }),
...(answer.sources ?? []).map((source) => ({
id: source.id,
text: source.content,
citation: citationFromSource(source),
+ source,
+ sourceStrength: source.source_strength,
})),
];
@@ -102,6 +123,13 @@ export function extractSafetyFindings(answer: RagAnswer | null | undefined, limi
for (const candidate of candidates) {
const text = conciseSourceText(candidate.text);
if (!text) continue;
+ if (answer.relevance) {
+ const sourceBacked = candidate.source?.relevance?.isSourceBacked;
+ const moderateOrStrong =
+ candidate.sourceStrength === "strong" || candidate.sourceStrength === "moderate";
+ const overlapsQuery = hasQueryConceptOverlap(text, coreTerms);
+ if (!sourceBacked && !(moderateOrStrong && overlapsQuery)) continue;
+ }
const match = safetyPatterns.find((item) => item.pattern.test(text));
if (!match) continue;
diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts
index e5f434cf9..745a70997 100644
--- a/src/lib/clinical-search.ts
+++ b/src/lib/clinical-search.ts
@@ -1,4 +1,51 @@
-import type { SearchResult } from "@/lib/types";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
+import type { RagQueryClass, SearchResult, SearchScoreExplanation } from "@/lib/types";
+
+type SearchIntent = "definition" | "protocol" | "drug_dosing" | "escalation_risk" | "document_lookup" | "general";
+
+type IntentSignals = {
+ intent: SearchIntent;
+ imageEvidenceFocus: boolean;
+ sectionedLookup: boolean;
+ hasDosingSignals: boolean;
+};
+
+export type RagQueryClassification = {
+ queryClass: RagQueryClass;
+ reasons: string[];
+ needsVisualEvidence: boolean;
+ needsSynthesis: boolean;
+};
+
+export const intentSignalWords = {
+ dosing: [
+ "dose",
+ "dosage",
+ "dosing",
+ "titrate",
+ "titration",
+ "mg",
+ "mcg",
+ "frequency",
+ "route",
+ "oral",
+ "intramuscular",
+ "im",
+ "po",
+ "prn",
+ "administer",
+ "administration",
+ "tablet",
+ "start",
+ "increase",
+ "cease",
+ ],
+ protocol: ["protocol", "procedure", "process", "workflow", "pathway", "step", "algorithm", "document", "guideline"],
+ escalation: ["escalat", "red flag", "urgent", "risk", "senior", "specialist", "review", "crisis", "rapid"],
+ visuals: ["table", "chart", "figure", "graph", "diagram", "flow", "matrix", "image"],
+ definitions: ["what is", "define", "definition", "describe", "meaning", "term", "short summary"],
+ lookup: ["document", "lookup", "find", "where", "chapter", "section", "title", "page"],
+} as const;
const textSearchStopWords = new Set([
"what",
@@ -6,6 +53,18 @@ const textSearchStopWords = new Set([
"where",
"which",
"how",
+ "please",
+ "can",
+ "you",
+ "me",
+ "about",
+ "find",
+ "search",
+ "look",
+ "lookup",
+ "now",
+ "today",
+ "exactly",
"the",
"and",
"with",
@@ -19,6 +78,9 @@ const textSearchStopWords = new Set([
"were",
"been",
"being",
+ "cover",
+ "covers",
+ "covered",
"managed",
"management",
"process",
@@ -43,6 +105,9 @@ const textSearchStopWords = new Set([
"patients",
"pts",
"clinical",
+ "psychiatric",
+ "mental",
+ "health",
]);
const synonymGroups = [
@@ -50,9 +115,56 @@ const synonymGroups = [
["contraindication", "avoid", "do not use", "caution", "exclusion"],
["escalation", "urgent", "senior review", "specialist review", "red flag"],
["adverse effect", "side effect", "toxicity", "safety-net", "warning"],
- ["dose", "dose limit", "maximum dose", "frequency", "route"],
+ ["dose", "dosing", "dose limit", "maximum dose", "frequency", "route", "oral", "intramuscular", "IM", "PRN"],
+];
+
+const intentPatterns: Array<{
+ intent: SearchIntent;
+ pattern: RegExp;
+ imageEvidenceFocus: boolean;
+ sectionedLookup: boolean;
+}> = [
+ {
+ intent: "definition",
+ pattern: /what\s+is|define|definition|meaning|term|describe|terminology/i,
+ imageEvidenceFocus: false,
+ sectionedLookup: false,
+ },
+ {
+ intent: "protocol",
+ pattern: /protocol|procedure|process|pathway|workflow|algorithm|guideline/i,
+ imageEvidenceFocus: false,
+ sectionedLookup: true,
+ },
+ {
+ intent: "drug_dosing",
+ pattern: /dose|dosage|dosing|titrate|mg|mcg|frequency|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer|table|chart|monitor/i,
+ imageEvidenceFocus: true,
+ sectionedLookup: false,
+ },
+ {
+ intent: "escalation_risk",
+ pattern: /escalat|urgent|red\s*flag|risk|senior|rapid|immediate|crisis/i,
+ imageEvidenceFocus: false,
+ sectionedLookup: true,
+ },
+ {
+ intent: "document_lookup",
+ pattern: /search|lookup|find|where|section|page|document title|document id/i,
+ imageEvidenceFocus: false,
+ sectionedLookup: true,
+ },
];
+const comparisonPattern = /\b(compare|compared|versus|vs|between|difference\w*|conflict\w*)\b/i;
+const tableThresholdPattern =
+ /\b(table|chart|matrix|threshold|cut[\s-]?off|cutoff|level|range|score|scale|criteria|criterion|anc|fbc|neutrophil|white cell|when to withhold|withhold|cease|stop|maximum|minimum|baseline)\b/i;
+const medicationDoseRiskPattern =
+ /\b(medication|medicine|pharmacolog\w*|prescrib\w*|dose|dosage|dosing|mg|mcg|titrate|route|oral|intramuscular|administer\w*|\bim\b|\bpo\b|\bprn\b|clozapine|lithium|neuroleptic|antipsychotic|benzodiazepine|injectable|agitation|arousal|side effect\w*|adverse|toxicity|contraindicat\w*|monitor\w*|risk|urgent|escalat\w*)\b/i;
+const documentLookupPattern =
+ /\b(search|lookup|find|where|which document|documents?|document title|document id|file|pdf|page|section|guideline|procedure|protocol|form)\b/i;
+const broadSummaryPattern = /\b(summary|summarise|summarize|overview|explain|outline|tell me about|what should be considered)\b/i;
+
function tokens(text: string) {
return text
.replace(/([a-z])([A-Z])/g, "$1 $2")
@@ -61,6 +173,227 @@ function tokens(text: string) {
.filter((token) => token.length > 2);
}
+function parseDateAsYearsAgo(value?: string | null) {
+ if (!value) return null;
+ const date = new Date(value);
+ if (Number.isNaN(date.getTime())) return null;
+ const now = new Date();
+ return Math.max(0, (now.getTime() - date.getTime()) / (1000 * 60 * 60 * 24 * 365));
+}
+
+function clamp(value: number) {
+ return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0;
+}
+
+function containsAny(value: string, values: readonly string[]) {
+ return values.some((item) => value.includes(item));
+}
+
+function normalizeQueryTokenForLookups(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/[^\w]+/g, " ")
+ .replace(/\b(\d+)\b/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function normalizedClinicalQueryTokens(query: string) {
+ return normalizedClinicalSearchTokens(query)
+ .map((token) => token.toLowerCase())
+ .filter((token) => token.length > 2);
+}
+
+function hasImageEvidenceNeed(query: string) {
+ return /table|chart|diagram|flowchart|figure|image|visual|dose card|medication chart/i.test(query);
+}
+
+function extractionQualityScore(result: SearchResult) {
+ const quality = result.source_metadata?.extraction_quality;
+ if (quality === "good") return 0.03;
+ if (quality === "partial") return -0.01;
+ if (quality === "poor") return -0.04;
+ return 0;
+}
+
+function evidenceDensityBoost(result: SearchResult, tokens: string[]) {
+ if (!tokens.length) return 0;
+ const haystack = normalizeQueryTokenForLookups(`${result.section_heading ?? ""} ${result.content}`).split(" ");
+ if (!haystack.length) return 0;
+ const lookup = new Set(haystack);
+ const hits = tokens.filter((token) => lookup.has(token)).length;
+ return Math.min(0.1, hits * 0.028);
+}
+
+export function hasDoseEvidenceSupport(result: SearchResult) {
+ const haystack =
+ `${result.section_heading ?? ""} ${result.content} ${(result.memory_cards ?? [])
+ .map((card) => `${card.title} ${card.content}`)
+ .join(" ")} ${(result.images ?? [])
+ .map((image) => `${image.tableTextSnippet ?? ""} ${image.caption ?? ""} ${image.tableTitle ?? ""}`)
+ .join(" ")}`.toLowerCase();
+ return /\b(?:dose|dosage|dosing|mg|mcg|microgram|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer\w*|titration|titrate|frequency|maximum|tablet|injection|antipsychotic|benzodiazepine|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test(
+ haystack,
+ );
+}
+
+function sectionDepthSignal(querySignal: IntentSignals, sectionHeading: string | null) {
+ if (!sectionHeading || !querySignal.sectionedLookup) return 0;
+ if (/(protocol|procedure|pathway|workflow|algorithm|escalat|risk|monitor)/i.test(sectionHeading)) return 0.035;
+ return 0;
+}
+
+export function classifyQueryIntent(query: string): IntentSignals {
+ const lowered = query.toLowerCase();
+ const match = intentPatterns.find((entry) => entry.pattern.test(query));
+ const hasDosingSignals = containsAny(lowered, intentSignalWords.dosing);
+ const hasEscalationSignals = containsAny(lowered, intentSignalWords.escalation);
+ const hasImageSignals = containsAny(lowered, intentSignalWords.visuals);
+
+ return {
+ intent: match?.intent ?? "general",
+ imageEvidenceFocus: Boolean(match?.imageEvidenceFocus) || hasImageSignals,
+ sectionedLookup: Boolean(match?.sectionedLookup),
+ hasDosingSignals: hasDosingSignals && !hasEscalationSignals,
+ };
+}
+
+export function classifyRagQuery(query: string): RagQueryClassification {
+ const reasons: string[] = [];
+ const normalized = query.trim();
+
+ if (comparisonPattern.test(normalized)) {
+ reasons.push("comparison_terms");
+ return {
+ queryClass: "comparison",
+ reasons,
+ needsVisualEvidence: tableThresholdPattern.test(normalized),
+ needsSynthesis: true,
+ };
+ }
+
+ if (tableThresholdPattern.test(normalized)) {
+ reasons.push("table_or_threshold_terms");
+ return {
+ queryClass: "table_threshold",
+ reasons,
+ needsVisualEvidence: true,
+ needsSynthesis: false,
+ };
+ }
+
+ if (medicationDoseRiskPattern.test(normalized)) {
+ reasons.push("medication_dose_or_risk_terms");
+ return {
+ queryClass: "medication_dose_risk",
+ reasons,
+ needsVisualEvidence: /table|chart|matrix|dose card/i.test(normalized),
+ needsSynthesis: /recommend|decide|should|manage|consider|contraindicat|interaction|risk/i.test(normalized),
+ };
+ }
+
+ if (documentLookupPattern.test(normalized)) {
+ reasons.push("document_lookup_terms");
+ return {
+ queryClass: "document_lookup",
+ reasons,
+ needsVisualEvidence: /page|table|chart|figure|image/i.test(normalized),
+ needsSynthesis: false,
+ };
+ }
+
+ if (broadSummaryPattern.test(normalized)) {
+ reasons.push("broad_summary_terms");
+ return {
+ queryClass: "broad_summary",
+ reasons,
+ needsVisualEvidence: /table|chart|figure|image/i.test(normalized),
+ needsSynthesis: true,
+ };
+ }
+
+ reasons.push("no_specific_rag_class_terms");
+ return {
+ queryClass: "unsupported_or_general",
+ reasons,
+ needsVisualEvidence: false,
+ needsSynthesis: true,
+ };
+}
+
+function imageEvidenceSignal(query: string, result: SearchResult) {
+ const queryTokens = normalizedClinicalSearchTokens(query).join(" ");
+ const hasImageQuery = /table|chart|flowchart|diagram|graph|appendix|figure|matrix/.test(queryTokens);
+ const images = result.images ?? [];
+ const hasClinicalImageEvidence = images.some(
+ (image) =>
+ isClinicalImageEvidence(image) &&
+ (image.image_type === "clinical_table" ||
+ image.image_type === "flowchart_algorithm" ||
+ image.image_type === "medication_chart" ||
+ image.sourceKind === "table_crop" ||
+ image.sourceKind === "diagram_crop"),
+ );
+ const hasPotentialImageEvidence = images.some(
+ (image) =>
+ isClinicalImageEvidence(image) &&
+ (image.sourceKind === "page_region" || image.sourceKind === "cover_page" || image.sourceKind === "embedded"),
+ );
+ if (hasClinicalImageEvidence && hasImageQuery) return 0.24;
+ if (hasClinicalImageEvidence) return 0.1;
+ if (hasPotentialImageEvidence && hasImageQuery) return 0.16;
+ return 0;
+}
+
+function sectionMatchBoost(query: string, result: SearchResult) {
+ const loweredQuery = query.toLowerCase();
+ if (!result.section_heading) return 0;
+ const sectionTokens = result.section_heading
+ .toLowerCase()
+ .split(/\W+/)
+ .filter((token) => token.length > 2);
+ const matches = sectionTokens.filter((token) => loweredQuery.includes(token));
+ return Math.min(0.12, matches.length * 0.03);
+}
+
+function documentMetadataBoost(query: string, result: SearchResult, normalizedTokens: string[]) {
+ if (!normalizedTokens.length) return 0;
+ const queryClass = classifyRagQuery(query).queryClass;
+ const labels = result.document_labels ?? [];
+ const summary = result.document_summary ?? "";
+ const labelText = labels
+ .map((label) => `${label.label} ${label.label_type}`)
+ .join(" ")
+ .toLowerCase();
+ const summaryText = summary.toLowerCase();
+ const titleText = `${result.title} ${result.file_name}`.toLowerCase();
+
+ const tokenHits = normalizedTokens.filter((token) => {
+ const singular = token.endsWith("s") && !token.endsWith("ss") ? token.slice(0, -1) : token;
+ return labelText.includes(token) || labelText.includes(singular) || summaryText.includes(token);
+ }).length;
+ const highConfidenceLabelHits = labels.filter((label) => {
+ const labelNormalized = normalizeQueryTokenForLookups(label.label);
+ return label.confidence >= 0.6 && normalizedTokens.some((token) => labelNormalized.includes(token));
+ }).length;
+ const exactMetadataPhrase =
+ normalizedTokens.length >= 2 &&
+ (labelText.includes(normalizedTokens.join(" ")) ||
+ summaryText.includes(normalizedTokens.join(" ")) ||
+ titleText.includes(normalizedTokens.join(" ")));
+ const classBoost =
+ queryClass === "document_lookup" && (highConfidenceLabelHits > 0 || exactMetadataPhrase)
+ ? 0.045
+ : queryClass === "table_threshold" && /threshold|table|chart|monitor|level|anc|fbc/.test(summaryText)
+ ? 0.04
+ : queryClass === "medication_dose_risk" &&
+ /medicat|dose|monitor|risk|toxicity|side effect|clozapine|lithium|neuroleptic/.test(summaryText)
+ ? 0.035
+ : 0;
+
+ return Math.min(0.18, tokenHits * 0.018 + highConfidenceLabelHits * 0.035 + (exactMetadataPhrase ? 0.05 : 0)) + classBoost;
+}
+
export function normalizedClinicalSearchTokens(query: string) {
return tokens(query)
.filter((token) => token.length > 2 && !textSearchStopWords.has(token))
@@ -95,26 +428,189 @@ export function expandClinicalQuery(query: string) {
return `${query} ${Array.from(additions).join(" ")}`;
}
-export function clinicalRankScore(query: string, result: SearchResult) {
+function roundScore(value: number) {
+ return Number(value.toFixed(4));
+}
+
+export function clinicalRankExplanation(query: string, result: SearchResult): SearchScoreExplanation {
const queryTokens = tokens(query);
+ const querySignal = classifyQueryIntent(query);
+ const queryClass = classifyRagQuery(query).queryClass;
+ const normalizedTokens = normalizedClinicalQueryTokens(query);
const haystack =
`${result.title} ${result.file_name} ${result.section_heading ?? ""} ${result.content}`.toLowerCase();
const base = result.hybrid_score ?? result.similarity;
const title = `${result.title} ${result.file_name}`.toLowerCase();
+ const titleTokenText = tokens(`${result.title} ${result.file_name}`).join(" ");
+ const titleTokens = new Set(tokens(`${result.title} ${result.file_name}`));
+ const normalizedQueryTokens = normalizedClinicalSearchTokens(query);
const exactTitleBoost = queryTokens.some((token) => title.includes(token)) ? 0.08 : 0;
+ const titleTokenMatches = normalizedQueryTokens.filter(
+ (token) => titleTokens.has(token) || titleTokenText.includes(token),
+ ).length;
+ const titleCoverageBoost = Math.min(0.18, titleTokenMatches * 0.045);
+ const titlePhraseBoost =
+ /\btreatment\s+team\b/i.test(query) && /\btreatment\s+team\s+process\b/.test(titleTokenText) ? 0.18 : 0;
const safetyQuery = /\b(urgent|red flag|contraindicat|avoid|escalat|toxicity|dose|monitor)\b/i.test(query);
const safetyContentBoost =
- safetyQuery && /\b(urgent|red flag|contraindicat|avoid|escalat|toxicity|maximum|monitor)\b/i.test(haystack)
+ safetyQuery && /\b(urgent|red flag|contraindicat|avoid|escalat|toxicity|maximum|monitor)\b/.test(haystack)
? 0.08
: 0;
const status = result.source_metadata?.document_status;
const validation = result.source_metadata?.clinical_validation_status;
const statusBoost = status === "current" ? 0.04 : status === "outdated" ? -0.08 : 0;
const validationBoost = validation === "approved" ? 0.04 : validation === "locally_reviewed" ? 0.025 : 0;
+ const publicationYearsAgo = parseDateAsYearsAgo(result.source_metadata?.publication_date);
+ const reviewYearsAgo = parseDateAsYearsAgo(result.source_metadata?.review_date);
+ const freshnessBoost =
+ publicationYearsAgo === null
+ ? 0
+ : publicationYearsAgo <= 1
+ ? 0.06
+ : publicationYearsAgo <= 3
+ ? 0.03
+ : publicationYearsAgo >= 8
+ ? -0.03
+ : 0;
+ const reviewBoost = reviewYearsAgo === null ? 0 : reviewYearsAgo <= 1 ? 0.03 : reviewYearsAgo >= 5 ? -0.02 : 0;
+ const imageBoost = imageEvidenceSignal(query, result);
+ const sectionBoost = sectionMatchBoost(query, result);
+ const sectionedLookupBoost = querySignal.sectionedLookup ? 0.02 : 0;
+ const extractionBoost = extractionQualityScore(result);
+ const dosingBoost =
+ querySignal.hasDosingSignals &&
+ hasDoseEvidenceSupport(result)
+ ? 0.09
+ : 0;
+ const titleOnlyDosePenalty =
+ queryClass === "medication_dose_risk" &&
+ titleCoverageBoost >= 0.09 &&
+ !hasDoseEvidenceSupport(result)
+ ? -0.42
+ : 0;
+ const administrativeDoseQueryPenalty =
+ queryClass === "medication_dose_risk" &&
+ /\b(?:supporting information|relevant standards|references|document owner|review|authorisation|authorised by|published date|effective from|amendment)\b/i.test(
+ result.content,
+ ) &&
+ !/\b(?:mg|mcg|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|maximum dose|repeat(?:ing)? doses?|dose may be repeated|monitoring:)\b/i.test(
+ result.content,
+ )
+ ? -0.3
+ : 0;
+ const protocolBoost =
+ querySignal.intent === "protocol" && /(protocol|process|procedure|workflow|pathway|algorithm)/i.test(haystack)
+ ? 0.06
+ : 0;
+ const escalationBoost =
+ querySignal.intent === "escalation_risk" && /(urgent|escalat|risk|red flag|senior|specialist|admit)/i.test(haystack)
+ ? 0.05
+ : 0;
+ const definitionBoost =
+ querySignal.intent === "definition" && /(definition|meaning|term|describe|what is)/i.test(haystack) ? 0.05 : 0;
+ const evidenceBoost = evidenceDensityBoost(result, normalizedTokens);
+ const coreConceptTokens = normalizedTokens.filter(
+ (token) =>
+ ![
+ "dose",
+ "dosing",
+ "dosage",
+ "medication",
+ "medicine",
+ "route",
+ "oral",
+ "intramuscular",
+ "monitor",
+ "monitoring",
+ "risk",
+ ].includes(token),
+ );
+ const coreConceptPenalty =
+ queryClass === "medication_dose_risk" &&
+ coreConceptTokens.length > 0 &&
+ !coreConceptTokens.some((token) => haystack.includes(token))
+ ? -0.36
+ : 0;
+ const sectionDepth = sectionDepthSignal(querySignal, result.section_heading);
+ const metadataBoost = documentMetadataBoost(query, result, normalizedTokens);
+ const tableThresholdBoost =
+ queryClass === "table_threshold" &&
+ /(threshold|cut[\s-]?off|withhold|cease|stop|anc|fbc|table|chart|criteria|level|range|monitor)/i.test(haystack)
+ ? 0.06
+ : 0;
+ const comparisonCoverageBoost =
+ queryClass === "comparison" && titleCoverageBoost > 0 && evidenceBoost > 0.02 ? 0.025 : 0;
+ const routeSignal = (() => {
+ if (result.source_metadata?.document_status === "review_due") return -0.01;
+ if (result.source_metadata?.document_status === "outdated") return -0.04;
+ return 0;
+ })();
+
+ const titleBoost = exactTitleBoost + titleCoverageBoost + titlePhraseBoost;
+ const metadataSignals =
+ statusBoost + validationBoost + freshnessBoost + reviewBoost + extractionBoost + metadataBoost + routeSignal;
+ const clinicalSignalBoost =
+ safetyContentBoost +
+ imageBoost +
+ sectionBoost +
+ sectionedLookupBoost +
+ dosingBoost +
+ protocolBoost +
+ escalationBoost +
+ definitionBoost +
+ evidenceBoost +
+ tableThresholdBoost +
+ comparisonCoverageBoost +
+ sectionDepth;
+ const penalty = titleOnlyDosePenalty + administrativeDoseQueryPenalty + coreConceptPenalty;
+ const finalScore = clamp(base) + titleBoost + metadataSignals + clinicalSignalBoost + penalty;
- return base + exactTitleBoost + safetyContentBoost + statusBoost + validationBoost;
+ return {
+ vectorScore: roundScore(clamp(result.similarity)),
+ textRank: roundScore(result.text_rank ?? 0),
+ weightedHybridScore: roundScore(clamp(base)),
+ rrfScore: typeof result.rrf_score === "number" ? roundScore(result.rrf_score) : null,
+ memoryBoost: roundScore(result.memory_score ? Math.min(0.24, result.memory_score * 0.24) : 0),
+ titleBoost: roundScore(titleBoost),
+ metadataBoost: roundScore(metadataSignals),
+ clinicalSignalBoost: roundScore(clinicalSignalBoost),
+ penalty: roundScore(penalty),
+ finalScore: roundScore(finalScore),
+ strategy: "weighted_hybrid_served_rrf_telemetry",
+ };
+}
+
+export function clinicalRankScore(query: string, result: SearchResult) {
+ return clinicalRankExplanation(query, result).finalScore;
}
export function rankClinicalResults(query: string, results: SearchResult[]) {
- return [...results].sort((a, b) => clinicalRankScore(query, b) - clinicalRankScore(query, a));
+ const intent = classifyQueryIntent(query);
+ const wantsImageEvidence = hasImageEvidenceNeed(query) || intent.imageEvidenceFocus;
+ const ranked = [...results]
+ .map((result) => {
+ const explanation = clinicalRankExplanation(query, result);
+ const hasImageEvidence = (result.images ?? []).some((image) => isClinicalImageEvidence(image));
+ const imageEvidencePenalty = wantsImageEvidence && !hasImageEvidence ? -0.04 : 0;
+ const score = explanation.finalScore + imageEvidencePenalty;
+ return {
+ result,
+ explanation: {
+ ...explanation,
+ penalty: roundScore(explanation.penalty + imageEvidencePenalty),
+ finalScore: roundScore(score),
+ },
+ score,
+ };
+ })
+ .sort((a, b) => b.score - a.score)
+ .map((entry, index) => ({
+ ...entry.result,
+ score_explanation: {
+ ...entry.explanation,
+ finalRank: index + 1,
+ },
+ }));
+
+ return ranked;
}
diff --git a/src/lib/cross-document-synthesis.ts b/src/lib/cross-document-synthesis.ts
new file mode 100644
index 000000000..1d623d215
--- /dev/null
+++ b/src/lib/cross-document-synthesis.ts
@@ -0,0 +1,265 @@
+import type { RagQueryClass, SearchResult } from "@/lib/types";
+import { sourceTextForModel } from "@/lib/source-text-sanitizer";
+
+export type CrossDocumentSynthesisPlan = {
+ enabled: boolean;
+ reason: "single_document" | "routine_multi_document" | "comparison" | "broad_summary" | "explicit_cross_document";
+ results: SearchResult[];
+ documentCount: number;
+ selectedDocumentCount: number;
+ selectedSourceCount: number;
+ maxPerDocument: number;
+};
+
+export type CrossDocumentFusionBrief = {
+ text: string;
+ documentCount: number;
+ bulletCount: number;
+ sourceChunkIds: string[];
+};
+
+const crossDocumentQueryPattern =
+ /\b(?:across|combine|combined|synthesi[sz]e|together|overall|all documents|these documents|different documents|multiple documents|several documents|from the documents)\b/i;
+
+const comparisonQueryPattern = /\b(?:compare|compared|versus|vs|between|difference\w*|conflict\w*)\b/i;
+
+function score(result: SearchResult) {
+ return result.hybrid_score ?? result.similarity ?? 0;
+}
+
+function documentCount(results: SearchResult[]) {
+ return new Set(results.map((result) => result.document_id)).size;
+}
+
+function normalizeText(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/([a-z])([A-Z])/g, "$1 $2")
+ .replace(/[^a-z0-9/%<>.]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function queryTokens(query: string) {
+ const stopWords = new Set([
+ "what",
+ "which",
+ "when",
+ "where",
+ "should",
+ "could",
+ "would",
+ "across",
+ "these",
+ "those",
+ "documents",
+ "document",
+ "guidance",
+ "summary",
+ "summarize",
+ "summarise",
+ "consider",
+ ]);
+ return Array.from(
+ new Set(
+ normalizeText(query)
+ .split(/\s+/)
+ .map((token) => (token.endsWith("s") && !token.endsWith("ss") ? token.slice(0, -1) : token))
+ .filter((token) => token.length > 2 && !stopWords.has(token)),
+ ),
+ );
+}
+
+function compact(value: string, limit: number) {
+ const normalized = value.replace(/\s+/g, " ").trim();
+ if (normalized.length <= limit) return normalized;
+ return `${normalized.slice(0, limit - 3).trim()}...`;
+}
+
+function candidateSentences(result: SearchResult) {
+ const memoryText = (result.memory_cards ?? [])
+ .slice(0, 4)
+ .map((card) => `${card.card_type}: ${card.content}`)
+ .join(". ");
+ return sourceTextForModel(`${memoryText}. ${result.section_heading ?? ""}. ${result.content}`)
+ .split(/\r?\n|(?<=[.!?])\s+/)
+ .map((sentence) => compact(sentence, 260))
+ .filter((sentence) => sentence.length >= 24 && !/\b(?:document owner|authorisation|references|uncontrolled when printed)\b/i.test(sentence));
+}
+
+function bestDocumentPoint(query: string, results: SearchResult[]) {
+ const tokens = queryTokens(query);
+ const candidates = results.flatMap((result) =>
+ candidateSentences(result).map((sentence) => {
+ const normalized = normalizeText(sentence);
+ const tokenHits = tokens.filter((token) => normalized.includes(token)).length;
+ const clinicalSignal = /\b(?:must|should|required|monitor|escalat|risk|dose|mg|mcg|threshold|urgent|review|withhold|cease|avoid|baseline|hours?|days?)\b/i.test(
+ sentence,
+ )
+ ? 1.25
+ : 0;
+ return {
+ result,
+ sentence,
+ score: tokenHits * 1.6 + clinicalSignal + score(result),
+ };
+ }),
+ );
+ return candidates.sort((a, b) => b.score - a.score || a.sentence.length - b.sentence.length)[0] ?? null;
+}
+
+function isCrossDocumentIntent(query: string, queryClass: RagQueryClass) {
+ return queryClass === "comparison" || queryClass === "broad_summary" || crossDocumentQueryPattern.test(query);
+}
+
+function planReason(query: string, queryClass: RagQueryClass, documents: number): CrossDocumentSynthesisPlan["reason"] {
+ if (documents <= 1) return "single_document";
+ if (queryClass === "comparison" || comparisonQueryPattern.test(query)) return "comparison";
+ if (queryClass === "broad_summary") return "broad_summary";
+ if (crossDocumentQueryPattern.test(query)) return "explicit_cross_document";
+ return "routine_multi_document";
+}
+
+export function balanceCrossDocumentResults(
+ results: SearchResult[],
+ options: { limit?: number; maxPerDocument?: number; minDocuments?: number } = {},
+) {
+ const limit = options.limit ?? 8;
+ const maxPerDocument = options.maxPerDocument ?? 2;
+ const minDocuments = options.minDocuments ?? Math.min(4, documentCount(results));
+ const byDocument = new Map();
+
+ for (const result of results) {
+ byDocument.set(result.document_id, [...(byDocument.get(result.document_id) ?? []), result]);
+ }
+ for (const [documentId, documentResults] of byDocument) {
+ byDocument.set(
+ documentId,
+ [...documentResults].sort((a, b) => score(b) - score(a) || a.chunk_index - b.chunk_index),
+ );
+ }
+
+ const selected: SearchResult[] = [];
+ const selectedIds = new Set();
+ const rankedDocuments = [...byDocument.entries()].sort(
+ ([, a], [, b]) => score(b[0]) - score(a[0]) || (b[0].text_rank ?? 0) - (a[0].text_rank ?? 0),
+ );
+
+ for (const [, documentResults] of rankedDocuments.slice(0, minDocuments)) {
+ const best = documentResults[0];
+ if (!best || selectedIds.has(best.id)) continue;
+ selected.push(best);
+ selectedIds.add(best.id);
+ if (selected.length >= limit) return selected;
+ }
+
+ const counts = new Map();
+ for (const result of selected) counts.set(result.document_id, (counts.get(result.document_id) ?? 0) + 1);
+
+ for (const result of results) {
+ if (selected.length >= limit) break;
+ if (selectedIds.has(result.id)) continue;
+ const count = counts.get(result.document_id) ?? 0;
+ if (count >= maxPerDocument) continue;
+ selected.push(result);
+ selectedIds.add(result.id);
+ counts.set(result.document_id, count + 1);
+ }
+
+ for (const result of results) {
+ if (selected.length >= limit) break;
+ if (selectedIds.has(result.id)) continue;
+ selected.push(result);
+ selectedIds.add(result.id);
+ }
+
+ return selected.sort((a, b) => {
+ const selectedScoreDiff = score(b) - score(a);
+ return selectedScoreDiff || a.title.localeCompare(b.title) || a.chunk_index - b.chunk_index;
+ });
+}
+
+export function buildCrossDocumentSynthesisPlan(
+ query: string,
+ results: SearchResult[],
+ queryClass: RagQueryClass,
+): CrossDocumentSynthesisPlan {
+ const documents = documentCount(results);
+ const reason = planReason(query, queryClass, documents);
+ const enabled = documents > 1 && isCrossDocumentIntent(query, queryClass);
+ const maxPerDocument = queryClass === "comparison" ? 2 : 2;
+ const limit = queryClass === "comparison" || queryClass === "broad_summary" ? 8 : 6;
+ const balanced = enabled
+ ? balanceCrossDocumentResults(results, {
+ limit,
+ maxPerDocument,
+ minDocuments: Math.min(documents, queryClass === "comparison" ? 5 : 4),
+ })
+ : results;
+
+ return {
+ enabled,
+ reason,
+ results: balanced,
+ documentCount: documents,
+ selectedDocumentCount: documentCount(balanced),
+ selectedSourceCount: balanced.length,
+ maxPerDocument,
+ };
+}
+
+export function buildCrossDocumentSourceGuide(results: SearchResult[]) {
+ const grouped = new Map; chunks: string[] }>();
+ for (const result of results) {
+ const existing = grouped.get(result.document_id) ?? {
+ title: result.title,
+ pages: new Set(),
+ chunks: [],
+ };
+ if (result.page_number) existing.pages.add(result.page_number);
+ existing.chunks.push(result.id);
+ grouped.set(result.document_id, existing);
+ }
+
+ if (grouped.size <= 1) return "";
+
+ return `Cross-document synthesis guide:
+${[...grouped.values()]
+ .map((document, index) => {
+ const pages = [...document.pages].sort((a, b) => a - b).join(", ") || "page unavailable";
+ return `${index + 1}. ${document.title}: use pages ${pages}; source chunks ${document.chunks.slice(0, 3).join(", ")}`;
+ })
+ .join("\n")}`;
+}
+
+export function buildCrossDocumentFusionBrief(query: string, results: SearchResult[]): CrossDocumentFusionBrief {
+ const byDocument = new Map();
+ for (const result of results) {
+ byDocument.set(result.document_id, [...(byDocument.get(result.document_id) ?? []), result]);
+ }
+
+ if (byDocument.size <= 1) {
+ return { text: "", documentCount: byDocument.size, bulletCount: 0, sourceChunkIds: [] };
+ }
+
+ const points = [...byDocument.values()]
+ .map((documentResults) => bestDocumentPoint(query, documentResults))
+ .filter((point): point is NonNullable => Boolean(point))
+ .sort((a, b) => score(b.result) - score(a.result));
+
+ const sourceChunkIds = points.map((point) => point.result.id);
+ const text = `Fast fused source brief:
+${points
+ .map((point, index) => {
+ const page = point.result.page_number ? `p.${point.result.page_number}` : "page unavailable";
+ return `${index + 1}. ${point.result.title} (${page}): ${point.sentence}`;
+ })
+ .join("\n")}`;
+
+ return {
+ text,
+ documentCount: byDocument.size,
+ bulletCount: points.length,
+ sourceChunkIds,
+ };
+}
diff --git a/src/lib/deep-memory.ts b/src/lib/deep-memory.ts
new file mode 100644
index 000000000..76fcff31a
--- /dev/null
+++ b/src/lib/deep-memory.ts
@@ -0,0 +1,577 @@
+import type { SupabaseClient } from "@supabase/supabase-js";
+import { buildClinicalTextSearchQuery, classifyRagQuery, normalizedClinicalSearchTokens } from "@/lib/clinical-search";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
+import { embedTexts } from "@/lib/openai";
+import { sourceTextForDisplay, sourceTextForModel } from "@/lib/source-text-sanitizer";
+import type {
+ ClinicalDocument,
+ DocumentMemoryCard,
+ DocumentMemoryCardType,
+ DocumentSectionMemory,
+ SearchResult,
+} from "@/lib/types";
+
+export const ragDeepMemoryVersion = "rag-deep-memory-v1" as const;
+
+type MemoryChunk = {
+ id: string;
+ document_id: string;
+ page_number: number | null;
+ chunk_index: number;
+ section_heading: string | null;
+ content: string;
+ image_ids?: string[] | null;
+ metadata?: Record | null;
+};
+
+type MemoryImage = {
+ id: string;
+ page_number: number | null;
+ caption: string | null;
+ image_type?: string | null;
+ labels?: string[] | null;
+ source_kind?: string | null;
+ clinical_relevance_score?: number | null;
+ metadata?: Record | null;
+};
+
+type MemoryDocument = Pick & {
+ metadata?: ClinicalDocument["metadata"];
+};
+
+type BuiltMemoryCard = Omit & {
+ section_index?: number;
+};
+
+type SectionInsertRow = Omit;
+
+const highYieldTerms = /\b(?:must|should|required|immediate|urgent|escalat\w*|withhold|cease|stop|discontinue\w*|contraindicat\w*|avoid|monitor\w*|dose|mg|mcg|mmol\/l|anc|fbc|wbc|neutrophil|clozapine|lithium|antipsychotic|benzodiazepine|risk|red flag|baseline|weekly|monthly|hours?|days?|weeks?)\b/i;
+const thresholdTerms = /\b(?:threshold|level|range|score|rating|criteria|anc|fbc|wbc|neutrophil|cease|withhold|stop|<|>|≤|≥)\b/i;
+const medicationTerms = /\b(?:clozapine|lithium|antipsychotic|benzodiazepine|olanzapine|lorazepam|diazepam|haloperidol|neuroleptic|dose|mg|mcg|oral|intramuscular|im\b|po\b|route|titrate)\b/i;
+const riskTerms = /\b(?:risk|urgent|escalat\w*|red flag|toxicity|adverse|side effect|contraindicat\w*|avoid|senior|specialist|crisis|emergency)\b/i;
+const workflowTerms = /\b(?:workflow|pathway|process|procedure|step|refer|review|document|record|complete|form|responsib\w*|follow up|appointment)\b/i;
+const boilerplateTerms = /\b(?:uncontrolled when printed|document control|version control|authorisation date|copyright|all rights reserved)\b/i;
+
+function metadataRecord(metadata: unknown): Record {
+ return metadata && typeof metadata === "object" && !Array.isArray(metadata)
+ ? { ...(metadata as Record) }
+ : {};
+}
+
+function compactText(value: string | null | undefined, limit = 420) {
+ const clean = sourceTextForModel(String(value ?? ""));
+ if (!clean) return "";
+ return clean.length <= limit ? clean : `${clean.slice(0, limit - 3).trim()}...`;
+}
+
+function normalizeLookup(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/([a-z])([A-Z])/g, "$1 $2")
+ .replace(/[^a-z0-9%/.<>≤≥]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function normalizedTerms(value: string) {
+ const terms = normalizedClinicalSearchTokens(value);
+ const numericTerms = normalizeLookup(value)
+ .split(/\s+/)
+ .filter((token) => /^\d+(?:\.\d+)?$|^<|^>|^≤|^≥/.test(token));
+ return Array.from(new Set([...terms, ...numericTerms])).slice(0, 28);
+}
+
+function headingFromContent(content: string) {
+ const firstLine = content
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .find((line) => line.length >= 4 && line.length <= 100 && !line.includes("[[IMAGE_DATA_START]]"));
+ if (!firstLine) return null;
+ if (/^\d+\.?\s+[A-Z]/.test(firstLine) || /^[A-Z][A-Za-z0-9\s,;:()[\]/-]+$/.test(firstLine)) {
+ return firstLine.replace(/[:.\s]+$/, "");
+ }
+ return null;
+}
+
+function extractionQualityForChunks(chunks: MemoryChunk[]): DocumentSectionMemory["extraction_quality"] {
+ const qualities = chunks.map((chunk) => metadataRecord(chunk.metadata).extraction_quality);
+ if (qualities.includes("poor")) return "poor";
+ if (qualities.includes("partial")) return "partial";
+ return "good";
+}
+
+function sectionHeadingForChunk(chunk: MemoryChunk) {
+ return chunk.section_heading || headingFromContent(chunk.content) || `Page ${chunk.page_number ?? "unknown"}`;
+}
+
+export function buildDocumentSections(args: {
+ document: MemoryDocument;
+ chunks: MemoryChunk[];
+}): SectionInsertRow[] {
+ const sorted = [...args.chunks].sort((a, b) => a.chunk_index - b.chunk_index);
+ const groups: MemoryChunk[][] = [];
+
+ for (const chunk of sorted) {
+ const previous = groups.at(-1);
+ const heading = normalizeLookup(sectionHeadingForChunk(chunk));
+ const previousHeading = previous?.length ? normalizeLookup(sectionHeadingForChunk(previous[0])) : "";
+ const pageGap =
+ previous?.length && chunk.page_number && previous.at(-1)?.page_number
+ ? chunk.page_number - (previous.at(-1)?.page_number ?? chunk.page_number)
+ : 0;
+ if (!previous || (heading && previousHeading && heading !== previousHeading) || pageGap > 1) {
+ groups.push([chunk]);
+ } else {
+ previous.push(chunk);
+ }
+ }
+
+ return groups.map((group, sectionIndex) => {
+ const heading = sectionHeadingForChunk(group[0]);
+ const pages = group.map((chunk) => chunk.page_number).filter((page): page is number => Boolean(page));
+ const combined = compactText(group.map((chunk) => chunk.content).join(" "), 520);
+ const tags = normalizedTerms(`${args.document.title} ${heading} ${combined}`).slice(0, 14);
+ return {
+ document_id: args.document.id,
+ owner_id: args.document.owner_id ?? null,
+ section_index: sectionIndex,
+ heading,
+ heading_path: [heading],
+ page_start: pages.length ? Math.min(...pages) : null,
+ page_end: pages.length ? Math.max(...pages) : null,
+ chunk_ids: group.map((chunk) => chunk.id),
+ summary: combined || `${heading}: indexed source section.`,
+ tags,
+ extraction_quality: extractionQualityForChunks(group),
+ metadata: {
+ rag_indexing_version: ragDeepMemoryVersion,
+ source_path: args.document.source_path ?? null,
+ },
+ };
+ });
+}
+
+function imageMetadataString(image: MemoryImage, key: string) {
+ const value = metadataRecord(image.metadata)[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function imageTextForCards(images: MemoryImage[]) {
+ return images
+ .filter((image) => isClinicalImageEvidence(image))
+ .map((image) =>
+ [
+ image.caption,
+ image.image_type,
+ image.source_kind,
+ imageMetadataString(image, "table_label"),
+ imageMetadataString(image, "table_title"),
+ imageMetadataString(image, "table_text"),
+ ...(image.labels ?? []),
+ ]
+ .filter(Boolean)
+ .join(" "),
+ )
+ .join("\n");
+}
+
+function splitCandidateStatements(text: string) {
+ const withoutImageBlocks = sourceTextForDisplay(text);
+ return withoutImageBlocks
+ .split(/\r?\n|(?<=[.!?])\s+/)
+ .map((line) => compactText(line, 360))
+ .filter((line) => line.length >= 24 && line.length <= 360 && !boilerplateTerms.test(line));
+}
+
+function tableRowsFromMarkdown(text: string) {
+ const rows = text
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line.includes("|") && !/^\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?$/.test(line));
+
+ if (rows.length < 2) return [];
+ const header = rows[0]
+ .split("|")
+ .map((cell) => cell.trim())
+ .filter(Boolean)
+ .join(" ");
+
+ return rows.slice(1).map((row) => {
+ const cells = row
+ .split("|")
+ .map((cell) => cell.trim())
+ .filter(Boolean);
+ return compactText(`${header} ${cells.join(" ")}`, 360);
+ });
+}
+
+function tableRowsFromImageTags(text: string) {
+ const rows: string[] = [];
+ for (const match of text.matchAll(/\[\[IMAGE_DATA_START\]\]\s*([\s\S]*?)\s*\[\[IMAGE_DATA_END\]\]/g)) {
+ const block = match[1] ?? "";
+ const tableText = block.match(/Table text:\s*([\s\S]*?)(?:;\s*Description:|$)/i)?.[1];
+ if (tableText) rows.push(...tableRowsFromMarkdown(tableText));
+ }
+ return rows;
+}
+
+function classifyStatement(statement: string): { type: DocumentMemoryCardType; score: number } | null {
+ let score = 0;
+ let type: DocumentMemoryCardType = "citation_anchor";
+
+ if (thresholdTerms.test(statement)) {
+ type = "threshold";
+ score += 0.3;
+ }
+ if (medicationTerms.test(statement)) {
+ type = type === "citation_anchor" ? "medication" : type;
+ score += 0.24;
+ }
+ if (riskTerms.test(statement)) {
+ type = type === "citation_anchor" ? "risk" : type;
+ score += 0.2;
+ }
+ if (workflowTerms.test(statement)) {
+ type = type === "citation_anchor" ? "workflow" : type;
+ score += 0.16;
+ }
+ if (highYieldTerms.test(statement)) score += 0.22;
+ if (/\d/.test(statement)) score += 0.08;
+ if (boilerplateTerms.test(statement)) score -= 0.25;
+
+ return score >= 0.24 ? { type, score: Math.min(0.96, score) } : null;
+}
+
+function titleForCard(type: DocumentMemoryCardType, document: MemoryDocument, context: string) {
+ const terms = normalizedTerms(context).slice(0, 5).join(" ");
+ const prefix =
+ type === "table_row"
+ ? "Table evidence"
+ : type === "threshold"
+ ? "Threshold"
+ : type === "medication"
+ ? "Medication point"
+ : type === "risk"
+ ? "Risk/escalation point"
+ : type === "workflow"
+ ? "Workflow step"
+ : type === "section_summary"
+ ? "Section summary"
+ : "Source fact";
+ return compactText(`${prefix}: ${terms || document.title}`, 120);
+}
+
+function createCard(args: {
+ document: MemoryDocument;
+ chunk?: MemoryChunk;
+ sectionIndex?: number;
+ type: DocumentMemoryCardType;
+ content: string;
+ confidence: number;
+ sourceImageIds?: string[];
+ metadata?: Record;
+}): BuiltMemoryCard {
+ const title = titleForCard(args.type, args.document, `${args.chunk?.section_heading ?? ""} ${args.content}`);
+ return {
+ document_id: args.document.id,
+ owner_id: args.document.owner_id ?? null,
+ section_id: null,
+ section_index: args.sectionIndex,
+ card_type: args.type,
+ title,
+ content: compactText(args.content, 640),
+ normalized_terms: normalizedTerms(`${title} ${args.content}`),
+ page_number: args.chunk?.page_number ?? null,
+ source_chunk_ids: args.chunk ? [args.chunk.id] : [],
+ source_image_ids: args.sourceImageIds ?? [],
+ confidence: Math.max(0.35, Math.min(0.99, args.confidence)),
+ metadata: {
+ rag_indexing_version: ragDeepMemoryVersion,
+ generated_by: "local-worker",
+ chunk_index: args.chunk?.chunk_index ?? null,
+ section_heading: args.chunk?.section_heading ?? null,
+ ...args.metadata,
+ },
+ };
+}
+
+function sectionIndexForChunk(sections: SectionInsertRow[], chunkId: string) {
+ return sections.find((section) => section.chunk_ids.includes(chunkId))?.section_index;
+}
+
+function dedupeCards(cards: BuiltMemoryCard[]) {
+ const seen = new Set();
+ const deduped: BuiltMemoryCard[] = [];
+ for (const card of cards) {
+ const key = `${card.card_type}:${normalizeLookup(card.content).slice(0, 260)}`;
+ if (seen.has(key)) continue;
+ seen.add(key);
+ deduped.push(card);
+ }
+ return deduped;
+}
+
+export function buildDocumentMemoryCards(args: {
+ document: MemoryDocument;
+ chunks: MemoryChunk[];
+ images?: MemoryImage[];
+ sections?: SectionInsertRow[];
+}) {
+ const cards: BuiltMemoryCard[] = [];
+ const sections = args.sections ?? buildDocumentSections({ document: args.document, chunks: args.chunks });
+ const imagesByPage = new Map();
+ for (const image of args.images ?? []) {
+ imagesByPage.set(image.page_number ?? null, [...(imagesByPage.get(image.page_number ?? null) ?? []), image]);
+ }
+
+ for (const section of sections) {
+ cards.push(
+ createCard({
+ document: args.document,
+ sectionIndex: section.section_index,
+ type: "section_summary",
+ content: `${section.heading}: ${section.summary}`,
+ confidence: 0.68,
+ metadata: { chunk_ids: section.chunk_ids, page_start: section.page_start, page_end: section.page_end },
+ }),
+ );
+ }
+
+ for (const chunk of args.chunks) {
+ const sectionIndex = sectionIndexForChunk(sections, chunk.id);
+ const pageImages = imagesByPage.get(chunk.page_number) ?? [];
+ const sourceImageIds = Array.from(new Set([...(chunk.image_ids ?? []), ...pageImages.map((image) => image.id)]));
+ const imageContext = imageTextForCards(pageImages);
+
+ for (const row of [...tableRowsFromMarkdown(chunk.content), ...tableRowsFromImageTags(chunk.content)]) {
+ cards.push(
+ createCard({
+ document: args.document,
+ chunk,
+ sectionIndex,
+ type: "table_row",
+ content: row,
+ confidence: 0.9,
+ sourceImageIds,
+ metadata: { extraction_source: "table_row" },
+ }),
+ );
+ }
+
+ for (const statement of splitCandidateStatements(`${chunk.content}\n${imageContext}`)) {
+ const classification = classifyStatement(statement);
+ if (!classification) continue;
+ cards.push(
+ createCard({
+ document: args.document,
+ chunk,
+ sectionIndex,
+ type: classification.type,
+ content: statement,
+ confidence: 0.55 + classification.score * 0.42,
+ sourceImageIds,
+ metadata: { extraction_source: "chunk_statement" },
+ }),
+ );
+ }
+ }
+
+ if (!cards.some((card) => card.source_chunk_ids.length > 0) && args.chunks.length > 0) {
+ const chunk = args.chunks[0];
+ cards.push(
+ createCard({
+ document: args.document,
+ chunk,
+ sectionIndex: sectionIndexForChunk(sections, chunk.id),
+ type: "citation_anchor",
+ content: chunk.content,
+ confidence: 0.45,
+ sourceImageIds: chunk.image_ids ?? [],
+ metadata: { extraction_source: "fallback_anchor" },
+ }),
+ );
+ }
+
+ return dedupeCards(cards).sort((a, b) => b.confidence - a.confidence);
+}
+
+function embeddingText(card: BuiltMemoryCard) {
+ return `${card.title}\n${card.card_type}\n${card.content}\nTerms: ${card.normalized_terms.join(", ")}`;
+}
+
+async function stampDeepMemoryVersion(args: { supabase: SupabaseClient; documentId: string }) {
+ const { error } = await args.supabase.rpc("stamp_document_deep_memory_version", {
+ p_document_id: args.documentId,
+ p_version: ragDeepMemoryVersion,
+ });
+ if (error) throw new Error(error.message);
+}
+
+export async function upsertDocumentDeepMemory(args: {
+ supabase: SupabaseClient;
+ document: MemoryDocument;
+ chunks: MemoryChunk[];
+ images?: MemoryImage[];
+}) {
+ if (args.chunks.length === 0) throw new Error("Cannot build deep memory for a document with no chunks.");
+
+ const sections = buildDocumentSections(args);
+ const cards = buildDocumentMemoryCards({ ...args, sections });
+ if (cards.length === 0) throw new Error("Deep memory generated no source-backed memory cards.");
+
+ await args.supabase.from("document_memory_cards").delete().eq("document_id", args.document.id);
+ await args.supabase.from("document_sections").delete().eq("document_id", args.document.id);
+
+ const { data: insertedSections, error: sectionError } = await args.supabase
+ .from("document_sections")
+ .insert(sections)
+ .select("id,section_index");
+ if (sectionError) throw new Error(sectionError.message);
+
+ const sectionIds = new Map();
+ for (const section of insertedSections ?? []) {
+ sectionIds.set(section.section_index, section.id);
+ }
+
+ const embeddings = await embedTexts(cards.map(embeddingText));
+ if (embeddings.length !== cards.length) throw new Error("OpenAI returned an unexpected memory-card embedding count.");
+
+ for (let start = 0; start < cards.length; start += 100) {
+ const batch = cards.slice(start, start + 100).map((card, index) => {
+ const { section_index: sectionIndex, ...row } = card;
+ return {
+ ...row,
+ section_id: sectionIndex === undefined ? null : (sectionIds.get(sectionIndex) ?? null),
+ embedding: embeddings[start + index],
+ };
+ });
+ const { error } = await args.supabase.from("document_memory_cards").insert(batch);
+ if (error) throw new Error(error.message);
+ }
+
+ await stampDeepMemoryVersion({ supabase: args.supabase, documentId: args.document.id });
+ return { sections, memoryCards: cards };
+}
+
+function scoreMemoryCardForQuery(query: string, card: Pick) {
+ const queryTokens = normalizedTerms(query);
+ if (queryTokens.length === 0) return 0;
+ const termSet = new Set(card.normalized_terms ?? []);
+ const content = normalizeLookup(`${card.title} ${card.content}`);
+ const hits = queryTokens.filter((token) => termSet.has(token) || content.includes(token)).length;
+ const coverage = hits / queryTokens.length;
+ const queryClass = classifyRagQuery(query).queryClass;
+ const classBoost =
+ queryClass === "table_threshold" && ["table_row", "threshold"].includes(card.card_type)
+ ? 0.18
+ : queryClass === "medication_dose_risk" && ["medication", "threshold", "risk"].includes(card.card_type)
+ ? 0.14
+ : queryClass === "document_lookup" && card.card_type === "section_summary"
+ ? 0.08
+ : queryClass === "broad_summary" && card.card_type === "section_summary"
+ ? 0.08
+ : 0;
+ return Math.min(1, coverage * 0.62 + (card.confidence ?? 0.5) * 0.2 + classBoost);
+}
+
+function memoryCardRetrievalScore(card: DocumentMemoryCard) {
+ const hybridScore = Number(card.metadata?.memory_hybrid_score);
+ if (Number.isFinite(hybridScore) && hybridScore > 0) return Math.min(1, hybridScore);
+ return Math.min(1, card.confidence ?? 0.5);
+}
+
+export async function fetchMemoryCardsForQuery(args: {
+ supabase: SupabaseClient;
+ query: string;
+ queryEmbedding?: number[];
+ ownerId?: string;
+ documentIds?: string[];
+ matchCount?: number;
+}) {
+ try {
+ if (args.queryEmbedding?.length) {
+ const { data, error } = await args.supabase.rpc("match_document_memory_cards_hybrid", {
+ query_embedding: args.queryEmbedding,
+ query_text: buildClinicalTextSearchQuery(args.query),
+ match_count: args.matchCount ?? 32,
+ min_similarity: 0.1,
+ document_filters: args.documentIds?.length ? args.documentIds : null,
+ owner_filter: args.ownerId ?? null,
+ });
+
+ if (!error && data?.length) {
+ return ((data ?? []) as Array)
+ .map((card) => ({
+ ...card,
+ confidence: Number(card.confidence ?? 0.5),
+ metadata: {
+ ...(card.metadata ?? {}),
+ memory_similarity: card.similarity,
+ memory_text_rank: card.text_rank,
+ memory_hybrid_score: card.hybrid_score,
+ memory_rrf_score: card.rrf_score,
+ },
+ }))
+ .slice(0, args.matchCount ?? 32);
+ }
+ }
+
+ let queryBuilder = args.supabase
+ .from("document_memory_cards")
+ .select(
+ "id,document_id,owner_id,section_id,card_type,title,content,normalized_terms,page_number,source_chunk_ids,source_image_ids,confidence,metadata",
+ )
+ .textSearch("search_tsv", buildClinicalTextSearchQuery(args.query), { type: "websearch", config: "english" })
+ .order("confidence", { ascending: false })
+ .limit(args.matchCount ?? 32);
+
+ if (args.ownerId) queryBuilder = queryBuilder.eq("owner_id", args.ownerId);
+ if (args.documentIds?.length) queryBuilder = queryBuilder.in("document_id", args.documentIds);
+
+ const { data, error } = await queryBuilder;
+ if (error) return [];
+
+ return ((data ?? []) as DocumentMemoryCard[])
+ .map((card) => ({ ...card, confidence: Number(card.confidence ?? 0.5) }))
+ .sort((a, b) => scoreMemoryCardForQuery(args.query, b) - scoreMemoryCardForQuery(args.query, a))
+ .slice(0, args.matchCount ?? 32);
+ } catch {
+ return [];
+ }
+}
+
+export function applyMemoryCardBoosts(query: string, results: SearchResult[], cards: DocumentMemoryCard[]) {
+ if (results.length === 0 || cards.length === 0) return results;
+ const cardsByChunk = new Map();
+ for (const card of cards) {
+ for (const chunkId of card.source_chunk_ids ?? []) {
+ cardsByChunk.set(chunkId, [...(cardsByChunk.get(chunkId) ?? []), card]);
+ }
+ }
+
+ return results.map((result) => {
+ const relatedCards = cardsByChunk.get(result.id) ?? [];
+ if (relatedCards.length === 0) return result;
+ const memoryScore = Math.max(
+ ...relatedCards.map((card) => Math.max(scoreMemoryCardForQuery(query, card), memoryCardRetrievalScore(card))),
+ );
+ const base = result.hybrid_score ?? result.similarity;
+ return {
+ ...result,
+ hybrid_score: Math.min(0.99, base + Math.min(0.24, memoryScore * 0.24)),
+ memory_score: Number(memoryScore.toFixed(4)),
+ memory_cards: relatedCards
+ .sort(
+ (a, b) =>
+ Math.max(scoreMemoryCardForQuery(query, b), memoryCardRetrievalScore(b)) -
+ Math.max(scoreMemoryCardForQuery(query, a), memoryCardRetrievalScore(a)),
+ )
+ .slice(0, 4),
+ };
+ });
+}
diff --git a/src/lib/demo-data.ts b/src/lib/demo-data.ts
index 9a318a80e..76e7d93aa 100644
--- a/src/lib/demo-data.ts
+++ b/src/lib/demo-data.ts
@@ -98,6 +98,21 @@ export const demoImages: Array([
"custom",
]);
+export const ragEnrichmentVersion = ragDeepMemoryVersion;
+
+function metadataRecord(metadata: unknown): Record {
+ return metadata && typeof metadata === "object" && !Array.isArray(metadata)
+ ? { ...(metadata as Record) }
+ : {};
+}
+
const summarySchema = {
type: "object",
additionalProperties: false,
@@ -56,7 +73,17 @@ const summarySchema = {
label: { type: "string" },
label_type: {
type: "string",
- enum: ["topic", "document_type", "medication", "risk", "setting", "workflow", "population", "service", "custom"],
+ enum: [
+ "topic",
+ "document_type",
+ "medication",
+ "risk",
+ "setting",
+ "workflow",
+ "population",
+ "service",
+ "custom",
+ ],
},
confidence: { type: "number" },
},
@@ -165,29 +192,25 @@ function inferLabels(document: Pick;
chunks: EnrichmentChunk[];
images: EnrichmentImage[];
}) {
- const sourceBlock = args.chunks
- .slice(0, 18)
+ const selected = selectCoverageAwarePromptChunks(args.chunks);
+ const coverage = buildIndexingCoverageProfile({ chunks: args.chunks, images: args.images });
+ const sourceBlock = selected.chunks
.map((chunk) => {
const page = chunk.page_number ? `page ${chunk.page_number}` : "page unavailable";
return [
`chunk_id: ${chunk.id}`,
`${page}; chunk ${chunk.chunk_index}; heading: ${chunk.section_heading ?? "none"}`,
- trimChunk(chunk.content),
+ compactPromptChunk(chunk.content),
].join("\n");
})
.join("\n\n---\n\n");
const imageBlock = args.images
- .slice(0, 12)
.map((image) => {
const labels = image.labels?.length ? ` labels=${image.labels.join(", ")}` : "";
return `image_id: ${image.id}; page ${image.page_number ?? "n/a"}; type=${image.image_type ?? "unclear"};${labels} caption=${image.caption ?? ""}`;
@@ -204,6 +227,8 @@ File: ${args.document.file_name}
Source path: ${args.document.source_path ?? "unknown"}
Text excerpts:
+${buildCoveragePromptNote({ profile: coverage, selectedChunkIds: selected.chunks.map((chunk) => chunk.id) })}
+
${sourceBlock || "No source text available."}
Image evidence:
@@ -222,16 +247,16 @@ function parseGeneratedSummary(raw: string, document: Pick;
+ document: Pick & {
+ metadata?: ClinicalDocument["metadata"];
+ };
chunks: EnrichmentChunk[];
images?: EnrichmentImage[];
}) {
@@ -278,8 +309,23 @@ export async function upsertDocumentEnrichment(args: {
};
}
- const sourceChunkIds = args.chunks.slice(0, 12).map((chunk) => chunk.id);
- const sourceImageIds = (args.images ?? []).slice(0, 12).map((image) => image.id);
+ const selectedPromptChunks = selectCoverageAwarePromptChunks(args.chunks);
+ const coverageProfile = buildIndexingCoverageProfile({ chunks: args.chunks, images: args.images ?? [] });
+ const sourceChunkIds = args.chunks.map((chunk) => chunk.id);
+ const sourceImageIds = (args.images ?? []).map((image) => image.id);
+ const enrichedAt = new Date().toISOString();
+ const generatedMetadata = {
+ generated_by: "local-worker",
+ rag_enrichment_version: ragEnrichmentVersion,
+ rag_indexing_version: ragEnrichmentVersion,
+ rag_memory_version: ragEnrichmentVersion,
+ enriched_at: enrichedAt,
+ };
+ const coverageMetadata = {
+ coverage_profile: coverageProfile,
+ enrichment_prompt_strategy: selectedPromptChunks.strategy,
+ enrichment_prompt_chunk_ids: selectedPromptChunks.chunks.map((chunk) => chunk.id),
+ };
const { data: summary, error: summaryError } = await args.supabase
.from("document_summaries")
@@ -292,9 +338,9 @@ export async function upsertDocumentEnrichment(args: {
source_chunk_ids: sourceChunkIds,
source_image_ids: sourceImageIds,
model: env.OPENAI_FAST_ANSWER_MODEL,
- metadata: { generated_by: "local-worker", label_count: enrichment.labels.length },
- generated_at: new Date().toISOString(),
- updated_at: new Date().toISOString(),
+ metadata: { ...generatedMetadata, ...coverageMetadata, label_count: enrichment.labels.length },
+ generated_at: enrichedAt,
+ updated_at: enrichedAt,
},
{ onConflict: "document_id" },
)
@@ -311,12 +357,29 @@ export async function upsertDocumentEnrichment(args: {
owner_id: args.document.owner_id ?? null,
...label,
source: "generated",
- metadata: { generated_by: "local-worker" },
+ metadata: generatedMetadata,
})),
);
if (labelsError) throw new Error(labelsError.message);
}
+ const { error: documentMetadataError } = await args.supabase
+ .from("documents")
+ .update({
+ metadata: {
+ ...metadataRecord(args.document.metadata),
+ rag_enrichment_version: ragEnrichmentVersion,
+ rag_indexing_version: ragEnrichmentVersion,
+ rag_memory_version: ragEnrichmentVersion,
+ rag_enrichment_updated_at: enrichedAt,
+ generated_label_count: enrichment.labels.length,
+ ...coverageMetadata,
+ },
+ })
+ .eq("id", args.document.id);
+
+ if (documentMetadataError) throw new Error(documentMetadataError.message);
+
return { summary: summary as DocumentSummary, labels: enrichment.labels };
}
@@ -331,7 +394,10 @@ function tokenSet(text: string) {
function compactSummary(summary?: string | null) {
if (!summary) return null;
- const clean = summary.replace(/\s+/g, " ").replace(/^[-*]\s*/g, "").trim();
+ const clean = summary
+ .replace(/\s+/g, " ")
+ .replace(/^[-*]\s*/g, "")
+ .trim();
return clean.length <= 220 ? clean : `${clean.slice(0, 217).trim()}...`;
}
@@ -341,7 +407,7 @@ type RelatedDocumentMetadataRow = {
summary: string | null;
};
-async function fetchRelatedDocumentMetadata(args: {
+export async function fetchRelatedDocumentMetadata(args: {
supabase: SupabaseClient;
ownerId?: string;
documentIds: string[];
@@ -424,7 +490,7 @@ export async function fetchRelatedDocuments(args: {
file_name: result.file_name,
best_pages: page ? [page] : [],
best_chunk_ids: [result.id],
- image_count: result.images?.filter((image) => image.searchable !== false).length ?? 0,
+ image_count: result.images?.filter((image) => isClinicalImageEvidence(image)).length ?? 0,
score,
});
continue;
@@ -432,17 +498,20 @@ export async function fetchRelatedDocuments(args: {
existing.score = Math.max(existing.score, score);
if (page && !existing.best_pages.includes(page)) existing.best_pages.push(page);
if (!existing.best_chunk_ids.includes(result.id)) existing.best_chunk_ids.push(result.id);
- existing.image_count += result.images?.filter((image) => image.searchable !== false).length ?? 0;
+ existing.image_count += result.images?.filter((image) => isClinicalImageEvidence(image)).length ?? 0;
}
const documentIds = Array.from(grouped.keys());
if (documentIds.length === 0) return [];
- const metadataRows = await fetchRelatedDocumentMetadata({
- supabase: args.supabase,
- ownerId: args.ownerId,
- documentIds,
- });
+ const [metadataRows, visualCounts] = await Promise.all([
+ fetchRelatedDocumentMetadata({
+ supabase: args.supabase,
+ ownerId: args.ownerId,
+ documentIds,
+ }),
+ fetchDocumentVisualCounts(args.supabase, documentIds),
+ ]);
const labelsByDocument = new Map();
const summariesByDocument = new Map();
@@ -458,6 +527,7 @@ export async function fetchRelatedDocuments(args: {
const docLabels = labelsByDocument.get(document.document_id) ?? [];
const matchingLabel = docLabels.find((label) => queryTokens.has(label.label.toLowerCase()));
const summary = summariesByDocument.get(document.document_id) ?? null;
+ const counts = visualCounts.get(document.document_id);
return {
document_id: document.document_id,
title: document.title,
@@ -466,7 +536,8 @@ export async function fetchRelatedDocuments(args: {
summary: compactSummary(summary),
best_pages: document.best_pages.slice(0, 5),
best_chunk_ids: document.best_chunk_ids.slice(0, 5),
- image_count: document.image_count,
+ image_count: Math.max(document.image_count, counts?.imageCount ?? 0),
+ table_count: counts?.tableCount ?? 0,
match_reason: matchingLabel
? `Matched label: ${matchingLabel.label}`
: `Matched ${document.best_chunk_ids.length} indexed passage${document.best_chunk_ids.length === 1 ? "" : "s"}`,
@@ -476,3 +547,46 @@ export async function fetchRelatedDocuments(args: {
.sort((a, b) => b.score - a.score)
.slice(0, args.limit ?? 6);
}
+
+export async function fetchDocumentVisualCounts(supabase: SupabaseClient, documentIds: string[]) {
+ const counts = new Map();
+ const uniqueIds = Array.from(new Set(documentIds));
+ if (uniqueIds.length === 0) return counts;
+
+ const { data, error } = await supabase
+ .from("document_images")
+ .select("document_id,source_kind,searchable,image_type,clinical_relevance_score,metadata")
+ .in("document_id", uniqueIds)
+ .neq("image_type", "logo_decorative");
+
+ if (error) throw new Error(error.message);
+
+ for (const documentId of uniqueIds) counts.set(documentId, { imageCount: 0, tableCount: 0 });
+ for (const row of data ?? []) {
+ const documentId = String(row.document_id);
+ const current = counts.get(documentId) ?? { imageCount: 0, tableCount: 0 };
+ if (isClinicalImageEvidence(row)) {
+ current.imageCount += 1;
+ if (row.source_kind === "table_crop") current.tableCount += 1;
+ }
+ counts.set(documentId, current);
+ }
+
+ return counts;
+}
+
+export function toDocumentMatch(document: RelatedDocument): DocumentMatch {
+ return {
+ document_id: document.document_id,
+ title: document.title,
+ file_name: document.file_name,
+ labels: document.labels,
+ summarySnippet: document.summary,
+ bestPages: document.best_pages,
+ bestChunkIds: document.best_chunk_ids,
+ imageCount: document.image_count,
+ tableCount: document.table_count ?? 0,
+ matchReason: document.match_reason,
+ score: document.score,
+ };
+}
diff --git a/src/lib/document-naming.ts b/src/lib/document-naming.ts
new file mode 100644
index 000000000..5ff5e344e
--- /dev/null
+++ b/src/lib/document-naming.ts
@@ -0,0 +1,216 @@
+export type SupabaseLike = {
+ from: (table: string) => {
+ select: (columns?: string) => {
+ eq: (column: string, value: unknown) => {
+ limit: (count: number) => PromiseLike<{ data: unknown; error: { message: string } | null }>;
+ };
+ };
+ };
+};
+
+export type DocumentNamePlan = {
+ title: string;
+ baseTitle: string;
+ originalTitle: string | null;
+ originalFileName: string;
+ duplicateIndex: number;
+ duplicateGroupKey: string;
+ duplicateReason: "none" | "same_title_or_filename";
+};
+
+type ExistingDocumentName = {
+ id?: string;
+ title?: string | null;
+ file_name?: string | null;
+ content_hash?: string | null;
+ metadata?: unknown;
+};
+
+const titleAbbreviations = new Map([
+ ["admin", "Administering"],
+ ["assoc", "Associated"],
+ ["clin", "Clinical"],
+ ["coord", "Coordination"],
+ ["doc", "Document"],
+ ["docs", "Documents"],
+ ["guid", "Guideline"],
+ ["gui", "Guideline"],
+ ["imi", "IMI"],
+ ["kb", "KB"],
+ ["mh", "Mental Health"],
+ ["mhsp", "MHSP"],
+ ["mgt", "Management"],
+ ["mgmt", "Management"],
+ ["mon", "Monitoring"],
+ ["monitor", "Monitoring"],
+ ["nocc", "NOCC"],
+ ["pharma", "Pharmacological"],
+ ["pharm", "Pharmacological"],
+ ["pres", "Prescribing"],
+ ["proc", "Procedure"],
+ ["pt", "Patient"],
+ ["pts", "Patients"],
+ ["tx", "Treatment"],
+]);
+
+function stripExtension(fileName: string) {
+ return fileName.replace(/\.[A-Za-z0-9]{1,12}$/, "");
+}
+
+function splitCamelCase(value: string) {
+ return value
+ .replace(/([a-z])([A-Z])/g, "$1 $2")
+ .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+}
+
+function cleanTitleInput(value: string) {
+ return splitCamelCase(value)
+ .replace(/%20/g, " ")
+ .replace(/_/g, " ")
+ .replace(/(\d)\.(\d)/g, "$1__DOT__$2")
+ .replace(/\.+/g, " - ")
+ .replace(/__DOT__/g, ".")
+ .replace(/[/\\]+/g, " ")
+ .replace(/[^\w\s()[\]&+,.:-]+/g, " ")
+ .replace(/\b[0-9a-f]{8,}\b/gi, " ")
+ .replace(/\b(?:copy|final|new|scan|scanned)\s*\(?\d*\)?$/i, " ")
+ .replace(/\s*-\s*/g, " - ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function smartenWord(word: string) {
+ const normalized = word.toLowerCase();
+ const mapped = titleAbbreviations.get(normalized);
+ if (mapped) return mapped;
+ if (/^[A-Z0-9]{2,}$/.test(word)) return word;
+ if (/^v?\d+(?:\.\d+)*$/i.test(word)) return word.toUpperCase();
+ return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+}
+
+export function smartDocumentTitle(input: string) {
+ const source = stripExtension(input).trim();
+ const cleaned = cleanTitleInput(source);
+ if (!cleaned) return "Untitled document";
+
+ const title = cleaned
+ .split(/(\s+-\s+|[()[\]&+,.:-])/)
+ .map((part) => {
+ if (!part.trim() || /^[()[\]&+,.:-]$/.test(part) || /^\s+-\s+$/.test(part)) return part;
+ return part
+ .split(/\s+/)
+ .map(smartenWord)
+ .join(" ");
+ })
+ .join("")
+ .replace(/\s+/g, " ")
+ .replace(/\s+([()[\]&+,.:])/g, "$1")
+ .replace(/([([])\s+/g, "$1")
+ .trim();
+
+ return title.length <= 140 ? title : `${title.slice(0, 137).trim()}...`;
+}
+
+export function documentTitleKey(value: string) {
+ return smartDocumentTitle(value)
+ .toLowerCase()
+ .replace(/\([^)]*\bcopy\s+\d+[^)]*\)$/i, "")
+ .replace(/\([^)]*\bduplicate\s+\d+[^)]*\)$/i, "")
+ .replace(/[^a-z0-9]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function exactTitleKey(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function metadataRecord(value: unknown): Record {
+ return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {};
+}
+
+function duplicateSuffixFromFileName(fileName: string) {
+ const stem = stripExtension(fileName);
+ const version = stem.match(/(?:^|[\s._-])(?:v|version)[\s._-]*(\d+(?:\.\d+){0,3})\b/i)?.[1];
+ if (version) return `v${version}`;
+
+ const isoDate = stem.match(/\b(20\d{2})[-_. ]?([01]\d)[-_. ]?([0-3]\d)\b/)?.slice(1, 4);
+ if (isoDate?.length === 3) return isoDate.join("-");
+
+ const auDate = stem.match(/\b([0-3]\d)[-_. ]([01]\d)[-_. ](20\d{2})\b/)?.slice(1, 4);
+ if (auDate?.length === 3) return `${auDate[2]}-${auDate[1]}-${auDate[0]}`;
+
+ return null;
+}
+
+function uniqueTitle(baseTitle: string, existingTitles: Set, preferredSuffix: string | null, duplicateIndex: number) {
+ const preferred = preferredSuffix ? `${baseTitle} (${preferredSuffix})` : `${baseTitle} (Copy ${duplicateIndex})`;
+ if (!existingTitles.has(exactTitleKey(preferred))) return preferred;
+
+ for (let index = Math.max(duplicateIndex, 2); index < 500; index += 1) {
+ const candidate = `${baseTitle} (Copy ${index})`;
+ if (!existingTitles.has(exactTitleKey(candidate))) return candidate;
+ }
+
+ return `${baseTitle} (${Date.now()})`;
+}
+
+export async function planDocumentName(args: {
+ supabase: SupabaseLike;
+ ownerId: string;
+ fileName: string;
+ requestedTitle?: string | null;
+ contentHash?: string | null;
+}): Promise {
+ const originalTitle = args.requestedTitle?.trim() || null;
+ const baseTitle = smartDocumentTitle(originalTitle || args.fileName);
+ const duplicateGroupKey = documentTitleKey(baseTitle);
+
+ const { data, error } = await args.supabase
+ .from("documents")
+ .select("id,title,file_name,content_hash,metadata")
+ .eq("owner_id", args.ownerId)
+ .limit(1000);
+ if (error) throw new Error(error.message);
+
+ const documents = Array.isArray(data) ? (data as ExistingDocumentName[]) : [];
+ const matching = documents.filter((document) => {
+ if (args.contentHash && document.content_hash === args.contentHash) return false;
+ const metadata = metadataRecord(document.metadata);
+ const groupKey =
+ typeof metadata.smart_title_group_key === "string" && metadata.smart_title_group_key.trim()
+ ? metadata.smart_title_group_key
+ : document.title
+ ? documentTitleKey(document.title)
+ : "";
+ return groupKey === duplicateGroupKey || documentTitleKey(document.file_name ?? "") === duplicateGroupKey;
+ });
+
+ if (matching.length === 0) {
+ return {
+ title: baseTitle,
+ baseTitle,
+ originalTitle,
+ originalFileName: args.fileName,
+ duplicateIndex: 1,
+ duplicateGroupKey,
+ duplicateReason: "none",
+ };
+ }
+
+ const existingTitles = new Set(documents.map((document) => exactTitleKey(document.title ?? "")).filter(Boolean));
+ const duplicateIndex = matching.length + 1;
+ return {
+ title: uniqueTitle(baseTitle, existingTitles, duplicateSuffixFromFileName(args.fileName), duplicateIndex),
+ baseTitle,
+ originalTitle,
+ originalFileName: args.fileName,
+ duplicateIndex,
+ duplicateGroupKey,
+ duplicateReason: "same_title_or_filename",
+ };
+}
diff --git a/src/lib/env.ts b/src/lib/env.ts
index 5a81a2449..d1465fa19 100644
--- a/src/lib/env.ts
+++ b/src/lib/env.ts
@@ -7,18 +7,25 @@ const envSchema = z.object({
SUPABASE_PROJECT_REF: z.string().optional(),
SUPABASE_PROJECT_NAME: z.string().optional(),
SUPABASE_SERVICE_ROLE_KEY: z.string().optional(),
+ NEXT_PUBLIC_LOCAL_NO_AUTH: z.enum(["true", "false"]).optional().default("false"),
+ LOCAL_NO_AUTH: z.enum(["true", "false"]).optional().default("false"),
+ LOCAL_NO_AUTH_OWNER_EMAIL: z.string().optional(),
+ LOCAL_NO_AUTH_OWNER_ID: z.string().optional(),
OPENAI_API_KEY: z.string().optional(),
OPENAI_EMBEDDING_MODEL: z.string().default("text-embedding-3-small"),
OPENAI_ANSWER_MODEL: z.string().default("gpt-5.4-mini"),
OPENAI_FAST_ANSWER_MODEL: z.string().default("gpt-5.4-mini"),
OPENAI_STRONG_ANSWER_MODEL: z.string().default("gpt-5.4"),
- OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(900),
+ OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(1400),
OPENAI_QUERY_CACHE_SIZE: z.coerce.number().int().nonnegative().default(200),
OPENAI_VISION_MODEL: z.string().default("gpt-5.4-mini"),
OPENAI_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(45000),
OPENAI_MAX_RETRIES: z.coerce.number().int().nonnegative().default(2),
OPENAI_PROMPT_CACHE_RETENTION: z.enum(["off", "in_memory", "24h"]).default("in_memory"),
- OPENAI_STORE_RESPONSES: z.enum(["true", "false"]).default("false").transform((value) => value === "true"),
+ OPENAI_STORE_RESPONSES: z
+ .enum(["true", "false"])
+ .default("false")
+ .transform((value) => value === "true"),
OPENAI_FAST_REASONING_EFFORT: z.enum(["none", "minimal", "low", "medium", "high"]).default("low"),
OPENAI_STRONG_REASONING_EFFORT: z.enum(["none", "minimal", "low", "medium", "high"]).default("medium"),
OPENAI_SUMMARY_REASONING_EFFORT: z.enum(["none", "minimal", "low", "medium", "high"]).default("low"),
@@ -28,7 +35,10 @@ const envSchema = z.object({
RAG_ANSWER_CACHE_SIZE: z.coerce.number().int().nonnegative().default(100),
RAG_SEARCH_CACHE_TTL_MS: z.coerce.number().int().nonnegative().default(60000),
RAG_SEARCH_CACHE_SIZE: z.coerce.number().int().nonnegative().default(200),
- RAG_AWAIT_QUERY_LOGS: z.enum(["true", "false"]).default("false").transform((value) => value === "true"),
+ RAG_AWAIT_QUERY_LOGS: z
+ .enum(["true", "false"])
+ .default("false")
+ .transform((value) => value === "true"),
SUPABASE_DOCUMENT_BUCKET: z.string().default("clinical-documents"),
SUPABASE_IMAGE_BUCKET: z.string().default("clinical-images"),
MAX_UPLOAD_MB: z.coerce.number().int().positive().default(150),
@@ -75,3 +85,10 @@ export function isDemoMode() {
projectCheck.status === "mismatch"
);
}
+
+export function isLocalNoAuthMode() {
+ const publicNoAuth = process.env.NEXT_PUBLIC_LOCAL_NO_AUTH === "true";
+ const serverNoAuth = typeof window === "undefined" && env.LOCAL_NO_AUTH === "true";
+
+ return process.env.NODE_ENV !== "production" && (publicNoAuth || serverNoAuth);
+}
diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts
new file mode 100644
index 000000000..f8a65932d
--- /dev/null
+++ b/src/lib/evidence-relevance.ts
@@ -0,0 +1,433 @@
+import { hasDoseEvidenceSupport, normalizedClinicalSearchTokens } from "@/lib/clinical-search";
+import { sourceTextForDisplay } from "@/lib/source-text-sanitizer";
+import type {
+ DocumentMatch,
+ EvidenceRelevance,
+ EvidenceRelevanceVerdict,
+ SearchResult,
+ SourceEvidenceRelevance,
+ SourceStrength,
+} from "@/lib/types";
+
+const genericQueryTerms = new Set([
+ "answer",
+ "available",
+ "clinical",
+ "document",
+ "evidence",
+ "guideline",
+ "indexed",
+ "information",
+ "issue",
+ "item",
+ "list",
+ "management",
+ "overview",
+ "passage",
+ "patient",
+ "policy",
+ "question",
+ "recommendation",
+ "review",
+ "shown",
+ "source",
+ "support",
+ "table",
+ "text",
+]);
+
+const namedMedicationTerms = new Set([
+ "amisulpride",
+ "aripiprazole",
+ "carbamazepine",
+ "clozapine",
+ "diazepam",
+ "droperidol",
+ "haloperidol",
+ "lamotrigine",
+ "lithium",
+ "lorazepam",
+ "olanzapine",
+ "paliperidone",
+ "promethazine",
+ "quetiapine",
+ "risperidone",
+ "valproate",
+ "zuclopenthixol",
+]);
+
+const verdictLabels: Record = {
+ direct: "Direct match",
+ partial: "Partial match",
+ nearby: "Nearby only",
+ none: "No direct indexed evidence",
+};
+
+function uniq(values: string[], limit = 8) {
+ return Array.from(new Set(values.filter(Boolean))).slice(0, limit);
+}
+
+function clamp(value: number, min = 0, max = 1) {
+ return Math.max(min, Math.min(max, value));
+}
+
+function normalizeTerm(term: string) {
+ const cleaned = term.toLowerCase().replace(/[^a-z0-9]+/g, "");
+ if (cleaned.endsWith("s") && !cleaned.endsWith("ss")) return cleaned.slice(0, -1);
+ return cleaned;
+}
+
+export function queryCoreTerms(query: string) {
+ const normalized = normalizedClinicalSearchTokens(query).map(normalizeTerm).filter(Boolean);
+ const raw = (query.toLowerCase().match(/[a-z0-9]+/g) ?? []).map(normalizeTerm).filter(Boolean);
+ const candidates = uniq([...normalized, ...raw], 14);
+ const specific = candidates.filter((term) => term.length >= 3 && !genericQueryTerms.has(term));
+ return specific.length ? specific.slice(0, 10) : candidates.filter((term) => term.length >= 3).slice(0, 10);
+}
+
+function textIncludesTerm(text: string, term: string) {
+ if (!term) return false;
+ if (term.length <= 3) return new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i").test(text);
+ return text.includes(term);
+}
+
+function normalizeSearchText(value: string) {
+ return sourceTextForDisplay(value).toLowerCase().replace(/[^a-z0-9]+/g, " ");
+}
+
+function labelsText(labels?: Array<{ label?: string | null; label_type?: string | null }>) {
+ return (labels ?? []).map((label) => `${label.label_type ?? ""} ${label.label ?? ""}`).join(" ");
+}
+
+function sourceTextBlocks(source: SearchResult) {
+ const title = normalizeSearchText(`${source.title} ${source.file_name} ${source.section_heading ?? ""}`);
+ const content = normalizeSearchText(
+ [
+ source.content,
+ source.adjacent_context ?? "",
+ ...(source.memory_cards ?? []).map((card) => `${card.title} ${card.content}`),
+ ...(source.images ?? []).map((image) =>
+ [
+ image.caption,
+ image.tableLabel,
+ image.tableTitle,
+ image.tableTextSnippet,
+ image.accessibleTableMarkdown,
+ image.labels?.join(" "),
+ ]
+ .filter(Boolean)
+ .join(" "),
+ ),
+ ].join(" "),
+ );
+ const metadata = normalizeSearchText(`${labelsText(source.document_labels)} ${source.document_summary ?? ""}`);
+ return { title, content, metadata, all: `${title} ${content} ${metadata}` };
+}
+
+function baseRankScore(source: SearchResult | DocumentMatch) {
+ const raw = "score" in source ? source.score : (source.score_explanation?.finalScore ?? source.hybrid_score ?? source.similarity ?? 0);
+ return clamp(Number.isFinite(raw) ? raw : 0);
+}
+
+function sourceStrengthBonus(strength: SourceStrength | undefined) {
+ if (strength === "strong") return 0.16;
+ if (strength === "moderate") return 0.08;
+ if (strength === "limited") return -0.05;
+ return 0;
+}
+
+function directnessVerdict(args: {
+ coreTerms: string[];
+ medicationTerms: string[];
+ matchedTerms: string[];
+ contentMatchedTerms: string[];
+ coverageScore: number;
+ rankScore: number;
+ strength?: SourceStrength;
+ doseQuery: boolean;
+ hasDoseEvidence: boolean;
+}) {
+ const medicationCovered =
+ args.medicationTerms.length === 0 || args.medicationTerms.every((term) => args.matchedTerms.includes(term));
+ const enoughContent = args.contentMatchedTerms.length >= Math.min(2, Math.max(1, args.coreTerms.length));
+ const enoughScore = args.rankScore >= 0.5 || args.strength === "strong" || args.strength === "moderate";
+ const hasDoseSupport = !args.doseQuery || args.hasDoseEvidence;
+
+ if (
+ args.coreTerms.length > 0 &&
+ medicationCovered &&
+ hasDoseSupport &&
+ args.coverageScore >= 0.72 &&
+ enoughContent &&
+ enoughScore
+ ) {
+ return "direct" satisfies EvidenceRelevanceVerdict;
+ }
+
+ if (
+ args.matchedTerms.length > 0 &&
+ medicationCovered &&
+ hasDoseSupport &&
+ (args.coverageScore >= 0.42 || args.contentMatchedTerms.length >= 2)
+ ) {
+ return "partial" satisfies EvidenceRelevanceVerdict;
+ }
+
+ if (args.matchedTerms.length > 0 || args.rankScore > 0) return "nearby" satisfies EvidenceRelevanceVerdict;
+ return "none" satisfies EvidenceRelevanceVerdict;
+}
+
+function supportReason(relevance: Pick) {
+ if (relevance.verdict === "direct") {
+ return `Matched core concepts: ${relevance.matchedTerms.slice(0, 4).join(", ")}.`;
+ }
+ if (relevance.verdict === "partial") {
+ return relevance.missingTerms.length
+ ? `Some query concepts are supported, but missing: ${relevance.missingTerms.slice(0, 4).join(", ")}.`
+ : "Some query concepts are supported by retrieved source text.";
+ }
+ if (relevance.verdict === "nearby") {
+ return relevance.matchedTerms.length
+ ? `Only adjacent concepts matched: ${relevance.matchedTerms.slice(0, 4).join(", ")}.`
+ : "Retrieved passages scored as nearby neighbors without direct concept coverage.";
+ }
+ return "No retrieved indexed passage covered the query concepts.";
+}
+
+function relevanceChips(relevance: Pick) {
+ const chips: string[] = [];
+ if (relevance.matchedTerms.length) chips.push(`matched: ${relevance.matchedTerms.slice(0, 3).join(", ")}`);
+ if (relevance.missingTerms.length) chips.push(`missing: ${relevance.missingTerms.slice(0, 3).join(", ")}`);
+ if (relevance.verdict === "direct") chips.push("direct evidence");
+ if (relevance.verdict === "partial") chips.push("partial support");
+ if (relevance.verdict === "nearby") chips.push("nearby only");
+ if (relevance.verdict === "none") chips.push("no direct support");
+ if (relevance.verdict === "nearby" || relevance.verdict === "none") chips.push("limited support");
+ return chips.slice(0, 4);
+}
+
+export function buildSourceRelevance(query: string, source: SearchResult): SourceEvidenceRelevance {
+ const coreTerms = queryCoreTerms(query);
+ const medicationTerms = coreTerms.filter((term) => namedMedicationTerms.has(term));
+ const blocks = sourceTextBlocks(source);
+ const titleMatchedTerms = coreTerms.filter((term) => textIncludesTerm(blocks.title, term));
+ const contentMatchedTerms = coreTerms.filter((term) => textIncludesTerm(blocks.content, term));
+ const metadataMatchedTerms = coreTerms.filter((term) => textIncludesTerm(blocks.metadata, term));
+ const matchedTerms = uniq([...contentMatchedTerms, ...titleMatchedTerms, ...metadataMatchedTerms], 10);
+ const missingTerms = coreTerms.filter((term) => !matchedTerms.includes(term));
+ const coverageScore = coreTerms.length ? matchedTerms.length / coreTerms.length : 0;
+ const contentCoverage = coreTerms.length ? contentMatchedTerms.length / coreTerms.length : 0;
+ const rankScore = baseRankScore(source);
+ const doseQuery = /\b(?:dose|dosing|dosage|mg|mcg|microgram|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|titrate|titration|maximum)\b/i.test(
+ query,
+ );
+ const verdict = directnessVerdict({
+ coreTerms,
+ medicationTerms,
+ matchedTerms,
+ contentMatchedTerms,
+ coverageScore,
+ rankScore,
+ strength: source.source_strength,
+ doseQuery,
+ hasDoseEvidence: hasDoseEvidenceSupport(source),
+ });
+ const score = clamp(coverageScore * 0.52 + contentCoverage * 0.23 + rankScore * 0.2 + sourceStrengthBonus(source.source_strength));
+ const partial: SourceEvidenceRelevance = {
+ verdict,
+ label: verdictLabels[verdict],
+ matchedTerms,
+ missingTerms,
+ directSourceCount: verdict === "direct" ? 1 : 0,
+ weakSourceCount: verdict === "nearby" || verdict === "none" ? 1 : 0,
+ score: Number(score.toFixed(3)),
+ supportReason: "",
+ isSourceBacked: verdict === "direct" || verdict === "partial",
+ coverageScore: Number(coverageScore.toFixed(3)),
+ rankScore: Number(rankScore.toFixed(3)),
+ titleMatchedTerms,
+ contentMatchedTerms,
+ metadataMatchedTerms,
+ chips: [],
+ };
+ partial.supportReason = supportReason(partial);
+ partial.chips = relevanceChips(partial);
+ return partial;
+}
+
+export function annotateSearchResults(query: string, results: SearchResult[]) {
+ return results.map((result) => ({
+ ...result,
+ relevance: result.relevance ?? buildSourceRelevance(query, result),
+ }));
+}
+
+export function buildEvidenceRelevance(query: string, results: SearchResult[]): EvidenceRelevance {
+ if (results.length === 0) {
+ return {
+ verdict: "none",
+ label: verdictLabels.none,
+ matchedTerms: [],
+ missingTerms: queryCoreTerms(query),
+ directSourceCount: 0,
+ weakSourceCount: 0,
+ score: 0,
+ supportReason: "No indexed passages were retrieved for the query.",
+ isSourceBacked: false,
+ };
+ }
+
+ const annotated = results.map((result) => result.relevance ?? buildSourceRelevance(query, result));
+ const directSourceCount = annotated.filter((item) => item.verdict === "direct").length;
+ const partialSourceCount = annotated.filter((item) => item.verdict === "partial").length;
+ const weakSourceCount = annotated.filter((item) => item.verdict === "nearby" || item.verdict === "none").length;
+ const matchedTerms = uniq(annotated.flatMap((item) => item.matchedTerms), 10);
+ const coreTerms = queryCoreTerms(query);
+ const missingTerms = coreTerms.filter((term) => !matchedTerms.includes(term));
+ const topScore = Math.max(0, ...annotated.map((item) => item.score));
+ const avgTopScore =
+ annotated
+ .slice()
+ .sort((a, b) => b.score - a.score)
+ .slice(0, 3)
+ .reduce((sum, item) => sum + item.score, 0) / Math.min(3, annotated.length);
+ const score = Number(Math.max(topScore, avgTopScore).toFixed(3));
+ const verdict: EvidenceRelevanceVerdict =
+ directSourceCount > 0 && missingTerms.length === 0
+ ? "direct"
+ : directSourceCount > 0 || partialSourceCount > 0
+ ? "partial"
+ : matchedTerms.length > 0 || results.length > 0
+ ? "nearby"
+ : "none";
+ const relevance: EvidenceRelevance = {
+ verdict,
+ label: verdictLabels[verdict],
+ matchedTerms,
+ missingTerms,
+ directSourceCount,
+ weakSourceCount,
+ score,
+ supportReason: "",
+ isSourceBacked: verdict === "direct" || verdict === "partial",
+ };
+ relevance.supportReason =
+ verdict === "direct"
+ ? `Direct indexed support found in ${directSourceCount} source${directSourceCount === 1 ? "" : "s"}.`
+ : verdict === "partial"
+ ? missingTerms.length
+ ? `Partial indexed support found; missing: ${missingTerms.slice(0, 4).join(", ")}.`
+ : "Partial indexed support found across retrieved sources."
+ : verdict === "nearby"
+ ? "Retrieved sources are weak or adjacent; treat them as nearby evidence only."
+ : "No direct indexed evidence was found.";
+ return relevance;
+}
+
+function buildDocumentText(document: DocumentMatch) {
+ return normalizeSearchText(
+ [
+ document.title,
+ document.file_name,
+ document.summarySnippet ?? "",
+ document.matchReason,
+ labelsText(document.labels),
+ ].join(" "),
+ );
+}
+
+function documentFallbackRelevance(query: string, document: DocumentMatch): SourceEvidenceRelevance {
+ const coreTerms = queryCoreTerms(query);
+ const haystack = buildDocumentText(document);
+ const matchedTerms = coreTerms.filter((term) => textIncludesTerm(haystack, term));
+ const missingTerms = coreTerms.filter((term) => !matchedTerms.includes(term));
+ const coverageScore = coreTerms.length ? matchedTerms.length / coreTerms.length : 0;
+ const rankScore = baseRankScore(document);
+ const verdict: EvidenceRelevanceVerdict =
+ coverageScore >= 0.75 && rankScore >= 0.5
+ ? "direct"
+ : coverageScore >= 0.38
+ ? "partial"
+ : matchedTerms.length || rankScore > 0
+ ? "nearby"
+ : "none";
+ const score = clamp(coverageScore * 0.65 + rankScore * 0.25);
+ const relevance: SourceEvidenceRelevance = {
+ verdict,
+ label: verdictLabels[verdict],
+ matchedTerms,
+ missingTerms,
+ directSourceCount: verdict === "direct" ? 1 : 0,
+ weakSourceCount: verdict === "nearby" || verdict === "none" ? 1 : 0,
+ score: Number(score.toFixed(3)),
+ supportReason: "",
+ isSourceBacked: verdict === "direct" || verdict === "partial",
+ coverageScore: Number(coverageScore.toFixed(3)),
+ rankScore: Number(rankScore.toFixed(3)),
+ titleMatchedTerms: matchedTerms,
+ contentMatchedTerms: [],
+ metadataMatchedTerms: matchedTerms,
+ chips: [],
+ };
+ relevance.supportReason = supportReason(relevance);
+ relevance.chips = relevanceChips(relevance);
+ return relevance;
+}
+
+function combineDocumentSourceRelevance(query: string, document: DocumentMatch, sources: SearchResult[]) {
+ if (sources.length === 0) return documentFallbackRelevance(query, document);
+ const sourceRelevances = sources.map((source) => source.relevance ?? buildSourceRelevance(query, source));
+ const directSourceCount = sourceRelevances.filter((item) => item.verdict === "direct").length;
+ const partialSourceCount = sourceRelevances.filter((item) => item.verdict === "partial").length;
+ const weakSourceCount = sourceRelevances.filter((item) => item.verdict === "nearby" || item.verdict === "none").length;
+ const coreTerms = queryCoreTerms(query);
+ const matchedTerms = uniq(sourceRelevances.flatMap((item) => item.matchedTerms), 10);
+ const missingTerms = coreTerms.filter((term) => !matchedTerms.includes(term));
+ const score = Number(Math.max(document.score, ...sourceRelevances.map((item) => item.score)).toFixed(3));
+ const verdict: EvidenceRelevanceVerdict =
+ directSourceCount > 0 && missingTerms.length === 0
+ ? "direct"
+ : directSourceCount > 0 || partialSourceCount > 0
+ ? "partial"
+ : matchedTerms.length || document.score > 0
+ ? "nearby"
+ : "none";
+ const relevance: SourceEvidenceRelevance = {
+ verdict,
+ label: verdictLabels[verdict],
+ matchedTerms,
+ missingTerms,
+ directSourceCount,
+ weakSourceCount,
+ score,
+ supportReason: "",
+ isSourceBacked: verdict === "direct" || verdict === "partial",
+ coverageScore: coreTerms.length ? Number((matchedTerms.length / coreTerms.length).toFixed(3)) : 0,
+ rankScore: Number(document.score.toFixed(3)),
+ titleMatchedTerms: uniq(sourceRelevances.flatMap((item) => item.titleMatchedTerms), 6),
+ contentMatchedTerms: uniq(sourceRelevances.flatMap((item) => item.contentMatchedTerms), 6),
+ metadataMatchedTerms: uniq(sourceRelevances.flatMap((item) => item.metadataMatchedTerms), 6),
+ chips: [],
+ };
+ relevance.supportReason = supportReason(relevance);
+ relevance.chips = relevanceChips(relevance);
+ return relevance;
+}
+
+export function annotateDocumentMatches(query: string, matches: DocumentMatch[], results: SearchResult[] = []) {
+ const annotatedResults = annotateSearchResults(query, results);
+ const byDocument = new Map();
+ for (const result of annotatedResults) {
+ const list = byDocument.get(result.document_id) ?? [];
+ list.push(result);
+ byDocument.set(result.document_id, list);
+ }
+ return matches.map((match) => ({
+ ...match,
+ relevance: match.relevance ?? combineDocumentSourceRelevance(query, match, byDocument.get(match.document_id) ?? []),
+ }));
+}
+
+export function weakEvidence(relevance: EvidenceRelevance | null | undefined) {
+ return !relevance?.isSourceBacked || relevance.verdict === "nearby" || relevance.verdict === "none";
+}
diff --git a/src/lib/evidence-tags.ts b/src/lib/evidence-tags.ts
new file mode 100644
index 000000000..32a52ee80
--- /dev/null
+++ b/src/lib/evidence-tags.ts
@@ -0,0 +1,75 @@
+function normalize(value: string) {
+ return value
+ .toLowerCase()
+ .replace(/[_-]+/g, " ")
+ .replace(/[^a-z0-9/ ]+/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function titleCase(value: string) {
+ const acronyms = new Set(["anc", "fbc", "ecg", "ipu", "nsaids", "covid"]);
+ return normalize(value)
+ .split(" ")
+ .filter(Boolean)
+ .map((word) => (acronyms.has(word) ? word.toUpperCase() : `${word[0]?.toUpperCase() ?? ""}${word.slice(1)}`))
+ .join(" ");
+}
+
+function canonicalTag(label: string, context: string) {
+ const normalized = normalize(label);
+ const normalizedContext = normalize(context);
+
+ if (!normalized) return "";
+ if (normalized === "roles" || normalized === "responsibilities") {
+ if (/\b(clozapine|care coordinator|psychiatrist|clinic|monitoring)\b/.test(normalizedContext)) {
+ return "Care team responsibilities";
+ }
+ return "Roles and responsibilities";
+ }
+ if (normalized === "role" || normalized === "responsibility") return "Roles and responsibilities";
+ if (normalized === "care coordinator" || normalized === "care coordination") return "Care coordination";
+ if (normalized === "psychiatrist" || normalized === "consultant psychiatrist") return "Psychiatrist review";
+ if (normalized === "clozapine monitoring") return "Clozapine monitoring";
+ if (normalized === "clozapine clinic") return "Clozapine clinic";
+ if (normalized === "administration clerical" || normalized === "administration") return "Clinic administration";
+ if (normalized === "blood test" || normalized === "blood tests") return "Blood test monitoring";
+ if (normalized === "dose" || normalized === "dosing") return "Dose adjustment";
+ if (normalized === "monitoring") return normalizedContext.includes("clozapine") ? "Clozapine monitoring" : "Monitoring";
+
+ return titleCase(normalized);
+}
+
+function tagPriority(tag: string) {
+ const normalized = normalize(tag);
+ if (/\b(clozapine|lithium|medication|dose|blood test|monitoring|anc|fbc)\b/.test(normalized)) return 0;
+ if (/\b(escalation|risk|urgent|review|psychiatrist)\b/.test(normalized)) return 1;
+ if (/\b(care coordination|care team|workflow|clinic)\b/.test(normalized)) return 2;
+ if (/\b(administration|document|version|authorisation|publication)\b/.test(normalized)) return 5;
+ return 3;
+}
+
+export function smartEvidenceTags(
+ labels: Array | null | undefined,
+ context = "",
+ limit = 5,
+) {
+ const rawLabels = (labels ?? []).filter((label): label is string => Boolean(label?.trim()));
+ const canonical = rawLabels
+ .map((label, index) => ({ tag: canonicalTag(label, context), index }))
+ .filter((item) => item.tag);
+ const hasCareTeamResponsibilities = canonical.some((item) => item.tag === "Care team responsibilities");
+ const deduped = new Map();
+
+ for (const item of canonical) {
+ if (hasCareTeamResponsibilities && item.tag === "Roles and responsibilities") continue;
+ const key = normalize(item.tag);
+ if (!deduped.has(key)) deduped.set(key, item);
+ }
+
+ return [...deduped.values()]
+ .sort((a, b) => tagPriority(a.tag) - tagPriority(b.tag) || a.index - b.index || a.tag.localeCompare(b.tag))
+ .slice(0, limit)
+ .map((item) => item.tag);
+}
+
diff --git a/src/lib/evidence.ts b/src/lib/evidence.ts
index 7cb24d5df..0964a9b87 100644
--- a/src/lib/evidence.ts
+++ b/src/lib/evidence.ts
@@ -1,4 +1,7 @@
import { citationFromResult, documentCitationHref } from "@/lib/citations";
+import { buildEvidenceRelevance } from "@/lib/evidence-relevance";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
+import { sourceTextForDisplay, sourceTextForModel } from "@/lib/source-text-sanitizer";
import type {
BestSourceRecommendation,
ConflictOrGap,
@@ -11,17 +14,26 @@ import type {
SourceStrength,
VisualEvidenceCard,
} from "@/lib/types";
-
-const imageTagPattern = /\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g;
+import type { ClinicalImageUseClass } from "@/lib/types";
export function normalizeEvidenceText(text: string) {
- return text
- .replace(imageTagPattern, (tag) => {
- const description = tag.match(/Description:\s*([\s\S]*?)\s*\[\[IMAGE_DATA_END\]\]/)?.[1];
- return description ? `Image evidence: ${description.trim()}` : "";
- })
- .replace(/\s+/g, " ")
- .trim();
+ return sourceTextForModel(text);
+}
+
+function imageMetadata(image: { metadata?: Record | null }) {
+ return image.metadata && typeof image.metadata === "object" ? image.metadata : {};
+}
+
+function metadataText(metadata: Record, key: string) {
+ const value = metadata[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function compactText(value: string | null, limit = 500) {
+ if (!value) return null;
+ const compact = value.replace(/\s+/g, " ").trim();
+ if (!compact) return null;
+ return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact;
}
export function sourceStrengthForSimilarity(similarity: number): SourceStrength {
@@ -41,7 +53,38 @@ function queryTokens(query: string) {
["vomiting", "diarrhoea", "dehydration", "tremor", "confusion", "ataxia"].forEach((token) => expanded.add(token));
}
if (tokens.some((token) => ["clozapine", "table", "image", "monitoring"].includes(token))) {
- ["fbc", "anc", "myocarditis", "metabolic", "constipation"].forEach((token) => expanded.add(token));
+ ["fbc", "wbc", "anc", "neutrophil", "myocarditis", "metabolic", "constipation"].forEach((token) =>
+ expanded.add(token),
+ );
+ }
+ if (tokens.some((token) => ["dose", "dosing", "dosage", "titrate", "titration", "mg", "route"].includes(token))) {
+ [
+ "dose",
+ "doses",
+ "dosing",
+ "medication",
+ "oral",
+ "intramuscular",
+ "im",
+ "po",
+ "prn",
+ "maximum",
+ "repeat",
+ "frequency",
+ "benzodiazepine",
+ "antipsychotic",
+ "olanzapine",
+ "lorazepam",
+ "haloperidol",
+ "droperidol",
+ "promethazine",
+ "diazepam",
+ ].forEach((token) => expanded.add(token));
+ }
+ if (tokens.some((token) => ["withhold", "withholding", "cease", "stop", "stopping"].includes(token))) {
+ ["cease", "ceased", "discontinue", "discontinued", "interrupt", "interruption", "red"].forEach((token) =>
+ expanded.add(token),
+ );
}
if (tokens.some((token) => ["risk", "escalate", "senior"].includes(token))) {
["intent", "attempt", "agitation", "supervision", "review"].forEach((token) => expanded.add(token));
@@ -59,20 +102,79 @@ function sentenceScore(sentence: string, tokens: Set) {
return score;
}
+function hasDoseSentenceEvidence(sentence: string) {
+ return /\b(?:dose|doses|dosage|dosing|mg|mcg|microgram|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|route|frequency|repeat|maximum|benzodiazepine|antipsychotic|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam|administer)\b/i.test(
+ sentence,
+ );
+}
+
+function lowValueSentencePenalty(sentence: string, query: string) {
+ const isDoseQuery = /\b(?:dose|doses|dosage|dosing|mg|mcg|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|titrate|titration)\b/i.test(
+ query,
+ );
+ let penalty = 0;
+ if (
+ isDoseQuery &&
+ /\b(?:supporting information|relevant standards|references|document owner|authorisation|authorised by|published date|effective from|amendment|polypharmacy and high dose antipsychotic prescribing procedure)\b/i.test(
+ sentence,
+ ) &&
+ !/\b(?:mg|mcg|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|repeat|maximum|administer|monitoring:)\b/i.test(sentence)
+ ) {
+ penalty += 5;
+ }
+ if (isDoseQuery && !hasDoseSentenceEvidence(sentence)) penalty += 2;
+ if (/^agitation and arousal:?\s+pharmacological management guideline\b/i.test(sentence) && !hasDoseSentenceEvidence(sentence)) {
+ penalty += 3;
+ }
+ return penalty;
+}
+
+function tableRowQuoteCandidates(content: string) {
+ const lines = content
+ .split(/\r?\n+/)
+ .map((line) => normalizeText(line))
+ .filter((line) => line.split(/\s+/).length >= 2);
+ const candidates: string[] = [];
+ let activeHeader: string | null = null;
+
+ for (const line of lines) {
+ if (/\bwbc\b/i.test(line) && /\bneutrophil\b/i.test(line) && /\boutcome\b/i.test(line)) {
+ activeHeader = line;
+ continue;
+ }
+ if (activeHeader && /\b(?:green|amber|red)\b/i.test(line) && /\b(?:continue|cease|required|blood tests?)\b/i.test(line)) {
+ candidates.push(`${activeHeader} ${line}`);
+ }
+ }
+
+ return candidates;
+}
+
function bestQuoteFromContent(content: string, query: string) {
const clean = normalizeText(content);
if (!clean) return "";
const tokens = queryTokens(query);
- const sentences = clean
+ const displayContent = sourceTextForDisplay(content) || clean;
+ const rawLineCandidates = displayContent
+ .split(/\r?\n+/)
+ .map((line) => normalizeText(line))
+ .filter((line) => line.split(/\s+/).length >= 3);
+ const sentenceCandidates = clean
.split(/(?<=[.!?])\s+/)
.map((sentence) => sentence.trim())
.filter(Boolean);
+ const sentences = Array.from(new Set([...tableRowQuoteCandidates(content), ...rawLineCandidates, ...sentenceCandidates]));
const best =
sentences
- .map((sentence) => ({ sentence, score: sentenceScore(sentence, tokens) }))
- .sort((a, b) => b.score - a.score || b.sentence.length - a.sentence.length)[0]?.sentence ?? clean;
+ .map((sentence) => {
+ const score = sentenceScore(sentence, tokens);
+ const lengthPenalty = sentence.length > 340 ? 6 : sentence.length > 260 ? 1.2 : sentence.length > 180 ? 0.4 : 0;
+ return { sentence, score, adjustedScore: score - lengthPenalty - lowValueSentencePenalty(sentence, query) };
+ })
+ .sort((a, b) => b.adjustedScore - a.adjustedScore || b.score - a.score || a.sentence.length - b.sentence.length)[0]
+ ?.sentence ?? clean;
if (best.length <= 340) return best;
return `${best.slice(0, 337).trim()}...`;
@@ -146,6 +248,7 @@ export function buildSmartPanel(query: string, results: SearchResult[]) {
const documentBreakdown = buildDocumentBreakdown(results, quoteCards);
const visualEvidence = buildVisualEvidence(results);
const bestSource = selectBestSourceRecommendation(results, quoteCards);
+ const relevance = buildEvidenceRelevance(query, results);
return {
query,
@@ -158,6 +261,7 @@ export function buildSmartPanel(query: string, results: SearchResult[]) {
evidenceSummary: buildEvidenceSummary(results, quoteCards),
sourceCoverage: buildSourceCoverage(results),
conflictsOrGaps: detectConflictsOrGaps(results),
+ relevance,
} satisfies SmartPanel;
}
@@ -167,8 +271,11 @@ export function selectBestSourceRecommendation(
): BestSourceRecommendation | null {
if (results.length === 0) return null;
- let best = results[0];
- for (const result of results.slice(1)) {
+ const candidates = results.some((result) => result.relevance?.isSourceBacked)
+ ? results.filter((result) => result.relevance?.isSourceBacked)
+ : results;
+ let best = candidates[0];
+ for (const result of candidates.slice(1)) {
const bestScore = best.hybrid_score ?? best.similarity;
const resultScore = result.hybrid_score ?? result.similarity;
if (resultScore > bestScore || (resultScore === bestScore && result.similarity > best.similarity)) {
@@ -189,8 +296,9 @@ export function selectBestSourceRecommendation(
snippet: snippet.length === 260 ? `${snippet.slice(0, 257).trim()}...` : snippet,
quote,
section_heading: best.section_heading,
- image_count: (best.images ?? []).filter((image) => image.searchable !== false).length,
+ image_count: (best.images ?? []).filter((image) => isClinicalImageEvidence(image)).length,
viewer_href: documentCitationHref(citation),
+ relevance: best.relevance,
};
}
@@ -216,20 +324,36 @@ export function buildSourceCoverage(results: SearchResult[]): SourceCoverage {
documents_used: documents.size,
pages,
strongest_similarity: strongest,
- has_images: results.some((source) => source.images?.some((image) => image.searchable !== false)),
+ has_images: results.some((source) => source.images?.some((image) => isClinicalImageEvidence(image))),
};
}
export function buildVisualEvidence(results: SearchResult[], limit = 8) {
const seen = new Set();
- const cards: VisualEvidenceCard[] = [];
+ const cards: Array = [];
for (const result of results) {
for (const image of result.images ?? []) {
- if (image.searchable === false || image.image_type === "logo_decorative") continue;
+ if (!isClinicalImageEvidence(image)) continue;
if (seen.has(image.id)) continue;
seen.add(image.id);
const pageNumber = image.page_number ?? result.page_number;
+ const metadata = imageMetadata(image);
+ const rawTableText = metadataText(metadata, "table_text") ?? image.accessibleTableMarkdown ?? null;
+ const tableText = image.tableTextSnippet ?? rawTableText ?? metadataText(metadata, "table_text_snippet");
+ const sourceKind = image.sourceKind ?? image.source_kind ?? metadataText(metadata, "source_kind");
+ const tableRole = image.tableRole ?? metadataText(metadata, "table_role");
+ const priority =
+ (sourceKind === "table_crop" ? 40 : 0) +
+ (tableRole === "clinical" ? 30 : tableRole === "admin" || tableRole === "reference" ? 8 : 0) +
+ (result.relevance?.verdict === "direct"
+ ? 36
+ : result.relevance?.verdict === "partial"
+ ? 18
+ : result.relevance?.verdict === "nearby"
+ ? -10
+ : 0) +
+ (image.clinical_relevance_score ?? 0) * 20;
cards.push({
id: `${result.id}:${image.id}`,
image_id: image.id,
@@ -244,13 +368,42 @@ export function buildVisualEvidence(results: SearchResult[], limit = 8) {
viewer_href: `/documents/${result.document_id}?page=${pageNumber ?? 1}&chunk=${result.id}`,
image_type: image.image_type,
clinical_relevance_score: image.clinical_relevance_score,
+ source_kind: sourceKind,
+ tableLabel: image.tableLabel ?? metadataText(metadata, "table_label"),
+ tableTitle: image.tableTitle ?? metadataText(metadata, "table_title"),
+ tableRole,
+ clinicalUseClass:
+ typeof metadata.clinical_use_class === "string"
+ ? (metadata.clinical_use_class as ClinicalImageUseClass)
+ : (image.clinicalUseClass ?? null),
+ clinicalUseReason:
+ typeof metadata.clinical_use_reason === "string"
+ ? metadata.clinical_use_reason
+ : (image.clinicalUseReason ?? null),
+ accessibleTableMarkdown:
+ typeof metadata.accessible_table_markdown === "string"
+ ? metadata.accessible_table_markdown
+ : (image.accessibleTableMarkdown ?? rawTableText),
+ tableRows: Array.isArray(metadata.table_rows) ? (metadata.table_rows as string[][]) : (image.tableRows ?? null),
+ tableColumns: Array.isArray(metadata.table_columns)
+ ? (metadata.table_columns as string[])
+ : (image.tableColumns ?? null),
+ tableTextSnippet: compactText(tableText),
labels: image.labels,
+ relevance: result.relevance,
+ priority,
});
- if (cards.length >= limit) return cards;
}
}
- return cards;
+ return cards
+ .sort((a, b) => b.priority - a.priority)
+ .slice(0, limit)
+ .map((card) => {
+ const { priority, ...publicCard } = card;
+ void priority;
+ return publicCard;
+ });
}
export function buildEvidenceSummary(results: SearchResult[], quoteCards: QuoteCard[] = []): EvidenceSummary {
@@ -298,17 +451,19 @@ export function detectConflictsOrGaps(results: SearchResult[]): ConflictOrGap[]
return gaps;
}
-export function diversifySearchResults(results: SearchResult[], limit = 12, maxPerDocument = 4) {
- const enriched = dedupeSearchResults(results)
- .map((result) => ({
- ...result,
- source_strength: result.source_strength ?? sourceStrengthForSimilarity(result.similarity),
- }))
- .sort((a, b) => {
+export function diversifySearchResults(results: SearchResult[], limit = 12, maxPerDocument = 4, preserveOrder = false) {
+ const enriched = dedupeSearchResults(results).map((result) => ({
+ ...result,
+ source_strength: result.source_strength ?? sourceStrengthForSimilarity(result.similarity),
+ }));
+
+ if (!preserveOrder) {
+ enriched.sort((a, b) => {
const aScore = a.hybrid_score ?? a.similarity;
const bScore = b.hybrid_score ?? b.similarity;
return bScore - aScore || b.similarity - a.similarity;
});
+ }
const documentCounts = new Map();
const selected: SearchResult[] = [];
diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts
index 2cf13f224..d32f65812 100644
--- a/src/lib/extractors/document.ts
+++ b/src/lib/extractors/document.ts
@@ -10,9 +10,10 @@ import type { ExtractedDocument } from "@/lib/types";
function runPythonPdfExtractor(filePath: string, outputDir: string) {
const scriptPath = path.join(process.cwd(), "worker", "python", "extract_pdf_assets.py");
+ const outputJsonPath = path.join(outputDir, "extract.json");
return new Promise((resolve, reject) => {
- const child = spawn(process.env.PYTHON_BIN || "python", [scriptPath, filePath, outputDir], {
+ const child = spawn(process.env.PYTHON_BIN || "python", [scriptPath, filePath, outputDir, outputJsonPath], {
cwd: process.cwd(),
stdio: ["ignore", "pipe", "pipe"],
});
@@ -25,14 +26,15 @@ function runPythonPdfExtractor(filePath: string, outputDir: string) {
child.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
- child.on("close", (code) => {
+ child.on("close", async (code) => {
if (code !== 0) {
reject(new Error(stderr || `PDF extractor exited with code ${code}`));
return;
}
try {
- resolve(JSON.parse(stdout) as ExtractedDocument);
+ const jsonPayload = await readFile(outputJsonPath, "utf8").catch(() => extractJsonFromStdout(stdout));
+ resolve(JSON.parse(jsonPayload) as ExtractedDocument);
} catch (error) {
reject(error);
}
@@ -40,6 +42,13 @@ function runPythonPdfExtractor(filePath: string, outputDir: string) {
});
}
+function extractJsonFromStdout(stdout: string) {
+ const start = stdout.indexOf("{");
+ const end = stdout.lastIndexOf("}");
+ if (start === -1 || end === -1 || end <= start) return stdout;
+ return stdout.slice(start, end + 1);
+}
+
async function extractPdf(buffer: Buffer) {
const tempRoot = await mkdtemp(path.join(tmpdir(), "clinical-kb-"));
const pdfPath = path.join(tempRoot, "document.pdf");
diff --git a/src/lib/http.ts b/src/lib/http.ts
index cb99c8735..ab457c42e 100644
--- a/src/lib/http.ts
+++ b/src/lib/http.ts
@@ -24,8 +24,17 @@ function publicErrorMessage(error: unknown, status: number) {
function logSafeError(error: unknown, status: number) {
if (process.env.NODE_ENV === "test") return;
const name = error instanceof Error ? error.name : typeof error;
+ const message = error instanceof Error ? error.message : String(error);
+ const stack = error instanceof Error ? error.stack : undefined;
const details = error instanceof PublicApiError ? error.details : undefined;
- console.error("API request failed", { status, name, code: details?.code, requestId: details?.requestId });
+ console.error("API request failed", {
+ status,
+ name,
+ message,
+ code: details?.code,
+ requestId: details?.requestId,
+ stack,
+ });
}
export function jsonError(error: unknown, status = 500) {
diff --git a/src/lib/image-filtering.ts b/src/lib/image-filtering.ts
index 11536a300..58b503213 100644
--- a/src/lib/image-filtering.ts
+++ b/src/lib/image-filtering.ts
@@ -1,5 +1,14 @@
import type { ExtractedImage, ImageEvidenceCategory } from "@/lib/types";
+export const clinicalImagePolicyVersion = "clinical-image-use-v1" as const;
+
+export type ClinicalImageUseClass =
+ | "clinical_evidence"
+ | "administrative"
+ | "reference"
+ | "decorative_or_empty"
+ | "ambiguous";
+
export type CheapImageFilterInput = {
bytesLength: number;
imageHash: string;
@@ -12,8 +21,193 @@ export type ClassifiedImage = {
searchable: boolean;
clinical_relevance_score: number;
skip_reason: string | null;
+ clinical_use_class?: ClinicalImageUseClass;
+ clinical_use_reason?: string | null;
+ clinical_signal_score?: number;
+ admin_signal_score?: number;
+};
+
+export type ImageUseAssessmentInput = {
+ imageType?: ImageEvidenceCategory | string | null;
+ searchable?: boolean | null;
+ clinicalRelevanceScore?: number | null;
+ sourceKind?: string | null;
+ tableRole?: string | null;
+ tableText?: string | null;
+ tableTitle?: string | null;
+ tableLabel?: string | null;
+ caption?: string | null;
+ labels?: string[] | null;
+ skipReason?: string | null;
+};
+
+export type ImageUseAssessment = {
+ clinical_use_class: ClinicalImageUseClass;
+ clinical_use_reason: string;
+ clinical_signal_score: number;
+ admin_signal_score: number;
+ searchable: boolean;
+ clinical_relevance_score: number;
};
+const clinicalEvidencePatterns = [
+ /\bpatient(?:'s|s)?\b/i,
+ /\bconsumer(?:'s|s)?\b/i,
+ /\bassessment\b/i,
+ /\bmanagement\b/i,
+ /\bmonitor(?:ing|ed)?\b/i,
+ /\bobservation(?:s)?\b/i,
+ /\bmedication(?:s)?\b/i,
+ /\bdose\b|\bmg\b|\bmcg\b|\bim\b|\bpo\b/i,
+ /\bthreshold\b|\bscore\b|\brating\b|\bscale\b|\brange\b/i,
+ /\brisk\b|\bred flag\b|\bescalat\w*\b|\burgent\b|\bemergency\b/i,
+ /\btreatment\b|\btherapy\b|\bprocedure\b|\bintervention\b/i,
+ /\bcontraindicat\w*\b|\bside effect\b|\badverse\b|\btoxicity\b/i,
+ /\bclozapine\b|\blithium\b|\bbenzodiazepine\b|\bantipsychotic\b|\bect\b/i,
+ /\bworkflow\b|\bpathway\b|\bstep\s+\d+\b|\brefer(?:ral)?\b/i,
+ /\bresponsib\w*\b(?=.*\b(?:patient|consumer|clinical|monitor|medication|dose|risk|escalat|assessment|treatment)\b)/i,
+];
+
+const adminPatterns = [
+ /\bauthori[sz]ed by\b/i,
+ /\bauthori[sz]ation date\b/i,
+ /\bpublished date\b/i,
+ /\bversion\b/i,
+ /\beffective from\b/i,
+ /\beffective to\b/i,
+ /\bamendment(?:\(s\))?\b/i,
+ /\bdocument owner\b/i,
+ /\bapproval\b|\bapproved by\b|\bendorsed\b/i,
+ /\breview date\b|\bnext review\b|\bsuperseded\b/i,
+ /\bsite\b(?=.*\boperational area\b)(?=.*\bapplicable to\b)/i,
+ /\boperational area\b/i,
+ /\bapplicable to\b/i,
+ /\bpolicy sponsor\b|\bcontact person\b/i,
+];
+
+const referencePatterns = [
+ /\breferences\b/i,
+ /\bbibliography\b/i,
+ /\blegislation\b/i,
+ /\brelevant standards\b/i,
+ /\bassociated documents\b/i,
+ /\bdocuments support\b/i,
+];
+
+function combinedText(input: ImageUseAssessmentInput) {
+ return [
+ input.tableRole,
+ input.tableLabel,
+ input.tableTitle,
+ input.tableText,
+ input.caption,
+ ...(input.labels ?? []),
+ ]
+ .filter(Boolean)
+ .join("\n")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function countPatternHits(text: string, patterns: RegExp[]) {
+ return patterns.reduce((count, pattern) => count + (pattern.test(text) ? 1 : 0), 0);
+}
+
+function clampedScore(value: unknown, fallback = 0) {
+ const number = Number(value);
+ return Number.isFinite(number) ? Math.min(Math.max(number, 0), 1) : fallback;
+}
+
+function reasonForClass(useClass: ClinicalImageUseClass, clinicalScore: number, adminScore: number) {
+ if (useClass === "clinical_evidence") return `clinical signals=${clinicalScore}; admin/reference signals=${adminScore}`;
+ if (useClass === "administrative") return `document-control/admin table signals=${adminScore}`;
+ if (useClass === "reference") return "reference or bibliography material";
+ if (useClass === "decorative_or_empty") return "decorative, empty, or explicitly non-searchable image";
+ return `ambiguous image evidence; clinical signals=${clinicalScore}; admin/reference signals=${adminScore}`;
+}
+
+export function assessClinicalImageUse(input: ImageUseAssessmentInput): ImageUseAssessment {
+ const text = combinedText(input);
+ const tableRole = String(input.tableRole ?? "").toLowerCase();
+ const imageType = String(input.imageType ?? "");
+ const sourceKind = String(input.sourceKind ?? "");
+ const clinicalScore = countPatternHits(text, clinicalEvidencePatterns);
+ const adminScore = countPatternHits(text, adminPatterns);
+ const referenceScore = countPatternHits(text, referencePatterns);
+ const modelScore = clampedScore(input.clinicalRelevanceScore, 0.4);
+ const modelSearchable = input.searchable !== false;
+
+ let useClass: ClinicalImageUseClass = "ambiguous";
+ if (imageType === "logo_decorative" || /logo|decorative|duplicate|tiny|header|footer|empty|small/i.test(input.skipReason ?? "")) {
+ useClass = "decorative_or_empty";
+ } else if (tableRole === "reference" || (referenceScore > 0 && clinicalScore < 2)) {
+ useClass = "reference";
+ } else if (tableRole === "admin" || (adminScore >= 2 && clinicalScore < 2)) {
+ useClass = "administrative";
+ } else if (clinicalScore >= 2 && adminScore < 3) {
+ useClass = "clinical_evidence";
+ } else if (tableRole === "clinical" && clinicalScore >= 1) {
+ useClass = "clinical_evidence";
+ } else if (modelSearchable && modelScore >= 0.78 && adminScore === 0 && referenceScore === 0) {
+ useClass = "clinical_evidence";
+ } else if (sourceKind !== "table_crop" && modelSearchable && modelScore >= 0.72 && adminScore === 0) {
+ useClass = "clinical_evidence";
+ }
+
+ const searchable = useClass === "clinical_evidence";
+ return {
+ clinical_use_class: useClass,
+ clinical_use_reason: reasonForClass(useClass, clinicalScore, adminScore + referenceScore),
+ clinical_signal_score: clinicalScore,
+ admin_signal_score: adminScore + referenceScore,
+ searchable,
+ clinical_relevance_score: searchable ? Math.max(modelScore, Math.min(0.95, 0.45 + clinicalScore * 0.1)) : 0,
+ };
+}
+
+function safeMetadata(value: unknown) {
+ return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {};
+}
+
+export function clinicalUseClassFromMetadata(metadata: unknown): ClinicalImageUseClass | null {
+ const value = safeMetadata(metadata).clinical_use_class;
+ return typeof value === "string" &&
+ ["clinical_evidence", "administrative", "reference", "decorative_or_empty", "ambiguous"].includes(value)
+ ? (value as ClinicalImageUseClass)
+ : null;
+}
+
+export function isClinicalImageEvidence(image: {
+ searchable?: boolean | null;
+ image_type?: string | null;
+ clinical_relevance_score?: number | null;
+ clinicalUseClass?: string | null;
+ tableRole?: string | null;
+ source_kind?: string | null;
+ sourceKind?: string | null;
+ metadata?: unknown;
+}) {
+ if (image.searchable === false || image.image_type === "logo_decorative") return false;
+ if (image.clinicalUseClass) return image.clinicalUseClass === "clinical_evidence";
+ const metadata = safeMetadata(image.metadata);
+ const useClass = clinicalUseClassFromMetadata(metadata);
+ if (useClass) return useClass === "clinical_evidence";
+ const assessment = assessClinicalImageUse({
+ imageType: image.image_type,
+ searchable: image.searchable,
+ clinicalRelevanceScore: image.clinical_relevance_score,
+ sourceKind: image.sourceKind ?? image.source_kind,
+ tableRole: image.tableRole ?? (typeof metadata.table_role === "string" ? metadata.table_role : null),
+ tableText:
+ typeof metadata.table_text === "string"
+ ? metadata.table_text
+ : typeof metadata.table_text_snippet === "string"
+ ? metadata.table_text_snippet
+ : null,
+ });
+ return assessment.clinical_use_class === "clinical_evidence";
+}
+
function bboxLooksLikeHeaderOrFooter(bbox: ExtractedImage["bbox"]) {
if (!bbox) return false;
const [, y0, , y1] = bbox;
@@ -46,6 +240,9 @@ export function cheapImageSkipReason(input: CheapImageFilterInput) {
}
export function classifiedImageSkipReason(classification: ClassifiedImage) {
+ if (classification.clinical_use_class && classification.clinical_use_class !== "clinical_evidence") {
+ return classification.skip_reason ?? classification.clinical_use_reason ?? "not clinically useful evidence";
+ }
if (classification.image_type === "logo_decorative") return classification.skip_reason ?? "logo or decorative mark";
if (!classification.searchable) return classification.skip_reason ?? "not clinically searchable";
if (classification.clinical_relevance_score < 0.18) return classification.skip_reason ?? "low clinical relevance";
diff --git a/src/lib/indexed-source-formatting.ts b/src/lib/indexed-source-formatting.ts
new file mode 100644
index 000000000..053c2b41f
--- /dev/null
+++ b/src/lib/indexed-source-formatting.ts
@@ -0,0 +1,222 @@
+export type IndexedTextBlock =
+ | { type: "heading"; id: string; text: string; level: "title" | "section" }
+ | { type: "paragraph"; id: string; text: string }
+ | { type: "list"; id: string; items: string[] }
+ | { type: "table"; id: string; caption: string; rows: string[][] };
+
+function compactInline(value: string) {
+ return value.replace(/\s+/g, " ").trim();
+}
+
+function isPageFooter(line: string) {
+ return /^page\s+\d+\s+of\s+\d+$/i.test(line.trim());
+}
+
+function isNumberedHeading(line: string) {
+ return /^\d{1,2}\.\s+\S/.test(line.trim()) && line.trim().length <= 96;
+}
+
+function isLikelyTitle(rawLine: string, line: string, index: number) {
+ if (index > 2) return false;
+ if (line.length < 8 || line.length > 96) return false;
+ if (/[.:;]$/.test(line)) return false;
+ return rawLine.search(/\S/) >= 6 || /^[A-Z][A-Za-z\s,&/()-]+$/.test(line);
+}
+
+function isBulletLine(line: string) {
+ return /^[•*-]\s+/.test(line.trim());
+}
+
+function cleanBullet(line: string) {
+ return compactInline(line.trim().replace(/^[•*-]\s+/, ""));
+}
+
+function startsDataRow(line: string) {
+ const trimmed = line.trim();
+ if (/^(?:[<>]=?|[≥≤]|\d+(?:\.\d+)?\s*(?:hours?|days?|weeks?|months?)\b)/i.test(trimmed)) return true;
+ return /^[A-Za-z]/.test(trimmed) && splitWideHeader(line).length >= 3;
+}
+
+function splitWideHeader(line: string) {
+ return line
+ .trim()
+ .split(/\s{2,}/)
+ .map(compactInline)
+ .filter(Boolean);
+}
+
+function tableHeaderSignal(line: string) {
+ const cells = splitWideHeader(line);
+ if (cells.length < 3) return false;
+ return /\b(?:dose|monitoring|blood|time|result|level|threshold|action|frequency|role|responsib|test)\b/i.test(
+ cells.join(" "),
+ );
+}
+
+function nearestColumnIndex(start: number, columnStarts: number[]) {
+ let bestIndex = 0;
+ let bestDistance = Number.POSITIVE_INFINITY;
+ for (let index = 0; index < columnStarts.length; index += 1) {
+ const distance = Math.abs(start - columnStarts[index]);
+ if (distance < bestDistance) {
+ bestDistance = distance;
+ bestIndex = index;
+ }
+ }
+ return bestIndex;
+}
+
+function appendCell(row: string[], index: number, value: string) {
+ const clean = compactInline(value);
+ if (!clean) return;
+ row[index] = compactInline([row[index], clean].filter(Boolean).join(" "));
+}
+
+function cellSlices(line: string, columnStarts: number[]) {
+ return columnStarts.map((start, index) => {
+ const end = columnStarts[index + 1] ?? line.length;
+ return compactInline(line.slice(start, end));
+ });
+}
+
+function parseFixedWidthTable(lines: string[], startIndex: number) {
+ const headerLine = lines[startIndex] ?? "";
+ const headers = splitWideHeader(headerLine);
+ if (headers.length < 3) return null;
+
+ const columnStarts = headers.map((header, index) => {
+ const previousStart = index === 0 ? 0 : undefined;
+ const found = headerLine.indexOf(header, previousStart);
+ return found >= 0 ? found : index * 24;
+ });
+ const tableRows: string[][] = [];
+ const headerCells = [...headers];
+ let row: string[] | null = null;
+ let index = startIndex + 1;
+
+ for (; index < lines.length; index += 1) {
+ const rawLine = lines[index] ?? "";
+ const line = rawLine.trim();
+ if (isPageFooter(line)) break;
+ if (!line) continue;
+ if (isNumberedHeading(line) && tableRows.length > 0) break;
+
+ const firstTextIndex = rawLine.search(/\S/);
+ const slices = cellSlices(rawLine, columnStarts);
+ const hasRowStart = startsDataRow(rawLine);
+
+ if (!row && !hasRowStart) {
+ const targetIndex = nearestColumnIndex(Math.max(firstTextIndex, 0), columnStarts);
+ appendCell(headerCells, targetIndex, line);
+ continue;
+ }
+
+ if (hasRowStart && slices.some(Boolean)) {
+ if (row && row.some(Boolean)) tableRows.push(row);
+ row = Array.from({ length: headers.length }, () => "");
+ const wideCells = splitWideHeader(rawLine);
+ const initialCells = wideCells.length >= headers.length ? wideCells.slice(0, headers.length) : slices;
+ initialCells.forEach((cell, cellIndex) => appendCell(row!, cellIndex, cell));
+ continue;
+ }
+
+ if (row) {
+ const targetIndex = nearestColumnIndex(Math.max(firstTextIndex, 0), columnStarts);
+ const wideCells = splitWideHeader(rawLine);
+ if (wideCells.length > 1) {
+ wideCells.forEach((cell, offset) => appendCell(row!, Math.min(targetIndex + offset, headers.length - 1), cell));
+ } else {
+ appendCell(row, targetIndex, line);
+ }
+ }
+ }
+
+ if (row && row.some(Boolean)) tableRows.push(row);
+ if (tableRows.length === 0) return null;
+
+ return {
+ block: {
+ type: "table" as const,
+ id: `table:${startIndex}`,
+ caption: "Extracted clinical table",
+ rows: [headerCells.map(compactInline), ...tableRows.map((cells) => cells.map(compactInline))],
+ },
+ nextIndex: index,
+ };
+}
+
+function paragraphFrom(lines: string[]) {
+ return compactInline(lines.join(" "));
+}
+
+export function parseIndexedSourceText(text: string): IndexedTextBlock[] {
+ const rawLines = text
+ .replace(/\r/g, "\n")
+ .split("\n")
+ .map((line) => line.replace(/\s+$/g, ""));
+ const blocks: IndexedTextBlock[] = [];
+ let index = 0;
+
+ while (index < rawLines.length) {
+ const rawLine = rawLines[index] ?? "";
+ const line = rawLine.trim();
+ if (!line || isPageFooter(line)) {
+ index += 1;
+ continue;
+ }
+
+ if (tableHeaderSignal(rawLine)) {
+ const parsedTable = parseFixedWidthTable(rawLines, index);
+ if (parsedTable) {
+ blocks.push(parsedTable.block);
+ index = parsedTable.nextIndex;
+ continue;
+ }
+ }
+
+ if (isNumberedHeading(line)) {
+ blocks.push({ type: "heading", id: `heading:${index}`, text: line, level: "section" });
+ index += 1;
+ continue;
+ }
+
+ if (blocks.length === 0 && isLikelyTitle(rawLine, line, index)) {
+ blocks.push({ type: "heading", id: `title:${index}`, text: line, level: "title" });
+ index += 1;
+ continue;
+ }
+
+ if (isBulletLine(line)) {
+ const items: string[] = [];
+ while (index < rawLines.length && isBulletLine(rawLines[index] ?? "")) {
+ const item = cleanBullet(rawLines[index] ?? "");
+ if (item) items.push(item);
+ index += 1;
+ }
+ if (items.length) blocks.push({ type: "list", id: `list:${index}:${items.length}`, items });
+ continue;
+ }
+
+ const paragraphLines: string[] = [];
+ while (index < rawLines.length) {
+ const nextRawLine = rawLines[index] ?? "";
+ const nextLine = nextRawLine.trim();
+ if (
+ !nextLine ||
+ isPageFooter(nextLine) ||
+ isNumberedHeading(nextLine) ||
+ isBulletLine(nextLine) ||
+ (paragraphLines.length > 0 && tableHeaderSignal(nextRawLine))
+ ) {
+ break;
+ }
+ paragraphLines.push(nextLine);
+ index += 1;
+ }
+
+ const paragraph = paragraphFrom(paragraphLines);
+ if (paragraph) blocks.push({ type: "paragraph", id: `paragraph:${index}:${paragraph.slice(0, 24)}`, text: paragraph });
+ }
+
+ return blocks;
+}
diff --git a/src/lib/indexing-coverage.ts b/src/lib/indexing-coverage.ts
new file mode 100644
index 000000000..3f0335541
--- /dev/null
+++ b/src/lib/indexing-coverage.ts
@@ -0,0 +1,127 @@
+type CoverageChunk = {
+ id: string;
+ page_number: number | null;
+ chunk_index: number;
+ section_heading: string | null;
+ content: string;
+};
+
+type CoverageImage = {
+ id: string;
+ page_number: number | null;
+ caption: string | null;
+};
+
+const highYieldCoveragePattern =
+ /\b(?:must|should|required|urgent|immediate|escalat\w*|risk|red flag|monitor\w*|dose|mg|mcg|mmol|anc|fbc|wbc|threshold|withhold|cease|stop|contraindicat\w*|workflow|refer|review|follow[- ]?up|responsib\w*)\b/i;
+
+function compactText(value: string, limit: number) {
+ const clean = value.replace(/\s+/g, " ").trim();
+ if (clean.length <= limit) return clean;
+ return `${clean.slice(0, Math.max(0, limit - 3)).trim()}...`;
+}
+
+function uniqueById(items: T[]) {
+ const seen = new Set();
+ const unique: T[] = [];
+ for (const item of items) {
+ if (seen.has(item.id)) continue;
+ seen.add(item.id);
+ unique.push(item);
+ }
+ return unique;
+}
+
+function spreadPick(items: T[], count: number) {
+ if (items.length <= count) return [...items];
+ if (count <= 1) return [items[0]];
+ return Array.from({ length: count }, (_, index) => items[Math.round((index * (items.length - 1)) / (count - 1))]);
+}
+
+function chunkScore(chunk: CoverageChunk) {
+ let score = 0;
+ if (chunk.section_heading) score += 2;
+ if (highYieldCoveragePattern.test(chunk.content)) score += 4;
+ if (/\|.+\|/.test(chunk.content) || /\[\[IMAGE_DATA_START\]\]/.test(chunk.content)) score += 3;
+ if (/\d/.test(chunk.content)) score += 1;
+ return score;
+}
+
+export function buildIndexingCoverageProfile(args: {
+ pageCount?: number | null;
+ chunks: CoverageChunk[];
+ images?: CoverageImage[];
+}) {
+ const sortedChunks = [...args.chunks].sort((a, b) => a.chunk_index - b.chunk_index);
+ const pagesWithChunks = Array.from(
+ new Set(sortedChunks.map((chunk) => chunk.page_number).filter((page): page is number => Number.isFinite(page))),
+ ).sort((a, b) => a - b);
+ const expectedPages = Array.from({ length: Math.max(0, Number(args.pageCount ?? 0)) }, (_, index) => index + 1);
+ const missingPages =
+ expectedPages.length > 0 ? expectedPages.filter((page) => !pagesWithChunks.includes(page)) : [];
+ const contentCharacters = sortedChunks.reduce((sum, chunk) => sum + chunk.content.length, 0);
+ const sectionHeadings = Array.from(new Set(sortedChunks.map((chunk) => chunk.section_heading).filter(Boolean)));
+
+ return {
+ chunk_count: sortedChunks.length,
+ image_count: args.images?.length ?? 0,
+ content_character_count: contentCharacters,
+ pages_with_chunks: pagesWithChunks,
+ page_coverage_count: pagesWithChunks.length,
+ expected_page_count: expectedPages.length || null,
+ missing_page_numbers: missingPages,
+ has_complete_page_chunk_coverage: expectedPages.length === 0 || missingPages.length === 0,
+ section_heading_count: sectionHeadings.length,
+ section_headings_sample: sectionHeadings.slice(0, 40),
+ };
+}
+
+export function selectCoverageAwarePromptChunks(chunks: CoverageChunk[], maxChunks = 36) {
+ const sorted = [...chunks].sort((a, b) => a.chunk_index - b.chunk_index);
+ if (sorted.length <= maxChunks) {
+ return {
+ chunks: sorted,
+ strategy: "all_chunks",
+ };
+ }
+
+ const byHeading = uniqueById(
+ sorted.filter((chunk) => chunk.section_heading || /^\d{1,2}\.?\s+[A-Z]/.test(chunk.content.trim())),
+ ).slice(0, Math.ceil(maxChunks * 0.25));
+ const highYield = uniqueById(
+ sorted
+ .map((chunk) => ({ chunk, score: chunkScore(chunk) }))
+ .filter((item) => item.score > 0)
+ .sort((a, b) => b.score - a.score || a.chunk.chunk_index - b.chunk.chunk_index)
+ .map((item) => item.chunk),
+ ).slice(0, Math.ceil(maxChunks * 0.45));
+ const spread = spreadPick(sorted, Math.max(6, maxChunks - byHeading.length - highYield.length));
+ const selected = uniqueById([...spread, ...byHeading, ...highYield])
+ .sort((a, b) => a.chunk_index - b.chunk_index)
+ .slice(0, maxChunks);
+
+ return {
+ chunks: selected,
+ strategy: "coverage_spread_high_yield_headings",
+ };
+}
+
+export function buildCoveragePromptNote(args: {
+ profile: ReturnType;
+ selectedChunkIds: string[];
+}) {
+ const omitted = Math.max(0, args.profile.chunk_count - args.selectedChunkIds.length);
+ return [
+ `Coverage: ${args.profile.chunk_count} indexed chunks across ${args.profile.page_coverage_count} page(s); ${args.profile.image_count} searchable image/table item(s).`,
+ args.profile.has_complete_page_chunk_coverage
+ ? "Page-to-chunk coverage is complete for the indexed document page count."
+ : `Missing page chunk coverage: ${args.profile.missing_page_numbers.slice(0, 30).join(", ")}.`,
+ omitted > 0
+ ? `Prompt excerpts are coverage-selected from the full stored index; ${omitted} chunk(s) remain indexed and retrievable through search/citations.`
+ : "All indexed chunks are included in this enrichment prompt.",
+ ].join("\n");
+}
+
+export function compactPromptChunk(content: string) {
+ return compactText(content, 1400);
+}
diff --git a/src/lib/local-project-guard.ts b/src/lib/local-project-guard.ts
new file mode 100644
index 000000000..5f1bf90bf
--- /dev/null
+++ b/src/lib/local-project-guard.ts
@@ -0,0 +1,130 @@
+import { NextResponse } from "next/server";
+import { appName, localProjectId, projectPortEnd, projectPortStart } from "../../scripts/local-server-utils.mjs";
+
+const localHosts = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);
+
+export type LocalProjectIdentityPayload = {
+ appName: string;
+ projectId: string;
+ identityPath: "/api/local-project-id";
+ localServer: {
+ currentUrl: string | null;
+ currentPort: number | null;
+ projectPortStart: number;
+ projectPortEnd: number;
+ safeLocalOrigin: boolean;
+ requestOrigin: string | null;
+ requestReferer: string | null;
+ unsafeLocalCaller: string | null;
+ };
+};
+
+function portFor(url: URL) {
+ const explicit = Number.parseInt(url.port, 10);
+ if (Number.isInteger(explicit)) return explicit;
+ if (url.protocol === "http:") return 80;
+ if (url.protocol === "https:") return 443;
+ return null;
+}
+
+export function isLocalUrl(url: URL) {
+ return localHosts.has(url.hostname.toLowerCase());
+}
+
+export function isManagedProjectPort(port: number | null) {
+ return port !== null && port >= projectPortStart && port <= projectPortEnd;
+}
+
+export function localProjectIdentityPayload(requestUrl: string): LocalProjectIdentityPayload {
+ const url = new URL(requestUrl);
+ const local = isLocalUrl(url);
+ const port = local ? portFor(url) : null;
+ const currentUrl = local && port ? `${url.protocol}//${url.hostname}:${port}` : null;
+
+ return {
+ appName,
+ projectId: localProjectId(process.cwd()),
+ identityPath: "/api/local-project-id",
+ localServer: {
+ currentUrl,
+ currentPort: port,
+ projectPortStart,
+ projectPortEnd,
+ safeLocalOrigin: !local || isManagedProjectPort(port),
+ requestOrigin: null,
+ requestReferer: null,
+ unsafeLocalCaller: null,
+ },
+ };
+}
+
+function unsafeLocalCallerFromHeader(value: string | null) {
+ if (!value) return null;
+ try {
+ const url = new URL(value);
+ if (!isLocalUrl(url)) return null;
+ return isManagedProjectPort(portFor(url)) ? null : url.origin;
+ } catch {
+ return null;
+ }
+}
+
+export function localProjectRequestIdentityPayload(request: Request): LocalProjectIdentityPayload {
+ const payload = localProjectIdentityPayload(request.url);
+ const requestOrigin = request.headers.get("origin");
+ const requestReferer = request.headers.get("referer");
+ const unsafeLocalCaller =
+ unsafeLocalCallerFromHeader(requestOrigin) ?? unsafeLocalCallerFromHeader(requestReferer);
+
+ return {
+ ...payload,
+ localServer: {
+ ...payload.localServer,
+ requestOrigin,
+ requestReferer,
+ unsafeLocalCaller,
+ safeLocalOrigin: payload.localServer.safeLocalOrigin && !unsafeLocalCaller,
+ },
+ };
+}
+
+export function isSafeLocalProjectRequest(request: Request) {
+ return localProjectRequestIdentityPayload(request).localServer.safeLocalOrigin;
+}
+
+export class UnsafeLocalProjectOriginError extends Error {
+ constructor(readonly payload: LocalProjectIdentityPayload) {
+ super(
+ `Local requests for ${payload.appName} must use a managed project port. Run npm run ensure and use the printed URL.`,
+ );
+ this.name = "UnsafeLocalProjectOriginError";
+ }
+}
+
+export function assertSafeLocalProjectRequest(request: Request) {
+ const payload = localProjectRequestIdentityPayload(request);
+ if (!payload.localServer.safeLocalOrigin) {
+ throw new UnsafeLocalProjectOriginError(payload);
+ }
+}
+
+export function unsafeLocalProjectResponse(payload: LocalProjectIdentityPayload) {
+ return NextResponse.json(
+ {
+ error: "Use the ensured Clinical KB local URL before calling this API.",
+ run: "npm run ensure",
+ identity: payload,
+ },
+ {
+ status: 409,
+ headers: {
+ "Cache-Control": "no-store",
+ "X-Clinical-KB-Local-Guard": "unsafe-local-origin",
+ },
+ },
+ );
+}
+
+export function localProjectOriginErrorResponse(error: UnsafeLocalProjectOriginError) {
+ return unsafeLocalProjectResponse(error.payload);
+}
diff --git a/src/lib/openai.ts b/src/lib/openai.ts
index 319e52042..0e562b46b 100644
--- a/src/lib/openai.ts
+++ b/src/lib/openai.ts
@@ -1,5 +1,6 @@
import OpenAI from "openai";
import { env, requireOpenAIEnv } from "@/lib/env";
+import { assessClinicalImageUse } from "@/lib/image-filtering";
import { PublicApiError } from "@/lib/http";
import type { ImageEvidenceCategory, OpenAITokenUsage } from "@/lib/types";
@@ -145,7 +146,9 @@ function defaultReasoningEffort(operation: OpenAIOperation, model: string): Open
if (!model.startsWith("gpt-5")) return "none";
switch (operation) {
case "answer":
- return model === env.OPENAI_STRONG_ANSWER_MODEL ? env.OPENAI_STRONG_REASONING_EFFORT : env.OPENAI_FAST_REASONING_EFFORT;
+ return model === env.OPENAI_STRONG_ANSWER_MODEL
+ ? env.OPENAI_STRONG_REASONING_EFFORT
+ : env.OPENAI_FAST_REASONING_EFFORT;
case "summary":
return env.OPENAI_SUMMARY_REASONING_EFFORT;
case "vision_caption":
@@ -189,7 +192,8 @@ function responseBody(
prompt_cache_key: resolved.promptCacheKey ?? promptCacheKeyFor(operation),
prompt_cache_retention: promptCacheRetention,
metadata: { operation },
- reasoning: supportsReasoning(resolved.model) && reasoningEffort !== "none" ? { effort: reasoningEffort } : undefined,
+ reasoning:
+ supportsReasoning(resolved.model) && reasoningEffort !== "none" ? { effort: reasoningEffort } : undefined,
text: Object.keys(textConfig).length > 0 ? textConfig : undefined,
};
}
@@ -268,7 +272,7 @@ export function mapOpenAIError(error: unknown, operation: OpenAIOperation) {
const requestId = getRequestId(error);
if (isTimeoutError(error) || status === 408) {
- return new PublicApiError("OpenAI timed out. Retry with a narrower question or fewer selected documents.", 504, {
+ return new PublicApiError("OpenAI timed out. Trying source-only fallback response.", 504, {
code,
requestId,
});
@@ -286,10 +290,14 @@ export function mapOpenAIError(error: unknown, operation: OpenAIOperation) {
}
if (code.startsWith("invalid_image") || code === "image_too_large" || code === "unsupported_image_media_type") {
- return new PublicApiError("OpenAI could not read one of the extracted images. Inspect the source file and retry.", 502, {
- code,
- requestId,
- });
+ return new PublicApiError(
+ "OpenAI could not read one of the extracted images. Inspect the source file and retry.",
+ 502,
+ {
+ code,
+ requestId,
+ },
+ );
}
if (status === 400) {
@@ -520,14 +528,47 @@ const imageClassificationSchema = {
type: ["string", "null"],
description: "Reason the image is not searchable, or null when searchable.",
},
+ clinical_use_class: {
+ type: "string",
+ enum: ["clinical_evidence", "administrative", "reference", "decorative_or_empty", "ambiguous"],
+ description: "Whether this is useful clinical evidence, document administration, reference material, decorative/empty, or ambiguous.",
+ },
+ clinical_use_reason: {
+ type: "string",
+ description: "Short reason for the usefulness class based only on visible/extracted content.",
+ },
+ clinical_signal_score: {
+ type: "number",
+ description: "Count-like score from 0 to 10 for patient-care signals such as medication, monitoring, thresholds, risk, escalation, or workflow.",
+ },
+ admin_signal_score: {
+ type: "number",
+ description: "Count-like score from 0 to 10 for authorisation, version, amendment, site/applicability, reference, or document-control signals.",
+ },
},
- required: ["image_type", "searchable", "clinical_relevance_score", "labels", "caption", "skip_reason"],
+ required: [
+ "image_type",
+ "searchable",
+ "clinical_relevance_score",
+ "labels",
+ "caption",
+ "skip_reason",
+ "clinical_use_class",
+ "clinical_use_reason",
+ "clinical_signal_score",
+ "admin_signal_score",
+ ],
};
function sanitizeImageLabels(labels: unknown) {
if (!Array.isArray(labels)) return [];
return labels
- .map((label) => String(label).trim().toLowerCase().replace(/[^\w -]+/g, ""))
+ .map((label) =>
+ String(label)
+ .trim()
+ .toLowerCase()
+ .replace(/[^\w -]+/g, ""),
+ )
.filter((label) => label.length > 1)
.slice(0, 6);
}
@@ -536,14 +577,31 @@ export async function classifyAndCaptionImageFromBase64(args: {
base64: string;
mimeType: string;
nearbyText?: string;
+ sourceKind?: string | null;
+ candidateType?: string | null;
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableText?: string | null;
}) {
+ const extractionContext = [
+ `Source kind: ${args.sourceKind ?? "unknown"}`,
+ args.candidateType ? `Candidate type: ${args.candidateType}` : null,
+ args.tableLabel ? `Table label: ${args.tableLabel}` : null,
+ args.tableTitle ? `Table title: ${args.tableTitle}` : null,
+ args.tableRole ? `Extractor table role: ${args.tableRole}` : null,
+ args.tableText ? `Extracted table text:\n${args.tableText.slice(0, 2500)}` : null,
+ `Nearby page text:\n${(args.nearbyText ?? "not available").slice(0, 3500)}`,
+ ]
+ .filter(Boolean)
+ .join("\n\n");
const input = [
{
role: "user",
content: [
{
type: "input_text",
- text: `Nearby page text:\n${args.nearbyText ?? "not available"}`,
+ text: extractionContext,
},
{
type: "input_image",
@@ -561,7 +619,12 @@ export async function classifyAndCaptionImageFromBase64(args: {
schemaName: "clinical_image_classification",
instructions:
"Classify an extracted clinical guideline image and write a concise caption. " +
+ "Use searchable=true only when the image/table directly supports patient care, assessment, medication, dose, monitoring, observations, thresholds, risks, escalation, workflow, or clinical responsibilities. " +
+ "Set clinical_use_class=administrative and searchable=false for authorisation/publication/version/effective-date/amendment/site/operational-area/applicable-to document-control tables, even when they mention mental health. " +
+ "Set clinical_use_class=reference and searchable=false for bibliography, references, legislation, standards, or associated-document lists. " +
+ "Role/responsibility tables are clinical only when the duties affect patient care, medication, monitoring, assessment, escalation, or clinical workflow; purely governance/service-director/document-control responsibility tables are administrative. " +
"Set searchable false for logos, repeated decorative marks, empty crops, or images without clinical information. " +
+ "Do not mark a text-heavy table crop as decorative solely because it has no illustration. " +
"Do not infer patient-specific advice.",
reasoningEffort: env.OPENAI_VISION_REASONING_EFFORT,
});
@@ -572,24 +635,60 @@ export async function classifyAndCaptionImageFromBase64(args: {
? (parsed.image_type as ImageEvidenceCategory)
: "unclear";
const clinicalScore = Number(parsed.clinical_relevance_score);
- const searchable = Boolean(parsed.searchable) && imageType !== "logo_decorative";
+ const assessment = assessClinicalImageUse({
+ imageType,
+ searchable: Boolean(parsed.searchable),
+ clinicalRelevanceScore: clinicalScore,
+ sourceKind: args.sourceKind,
+ tableRole: args.tableRole,
+ tableText: args.tableText,
+ tableTitle: args.tableTitle,
+ tableLabel: args.tableLabel,
+ caption: typeof parsed.caption === "string" ? parsed.caption : null,
+ labels: sanitizeImageLabels(parsed.labels),
+ skipReason: typeof parsed.skip_reason === "string" ? parsed.skip_reason : null,
+ });
return {
image_type: imageType,
- searchable,
- clinical_relevance_score: Number.isFinite(clinicalScore) ? Math.min(Math.max(clinicalScore, 0), 1) : 0.4,
+ searchable: assessment.searchable && imageType !== "logo_decorative",
+ clinical_relevance_score: assessment.clinical_relevance_score,
labels: sanitizeImageLabels(parsed.labels),
caption: String(parsed.caption || "").trim() || "Extracted source image.",
- skip_reason: typeof parsed.skip_reason === "string" && parsed.skip_reason.trim() ? parsed.skip_reason.trim() : null,
+ skip_reason:
+ assessment.searchable
+ ? typeof parsed.skip_reason === "string" && parsed.skip_reason.trim()
+ ? parsed.skip_reason.trim()
+ : null
+ : assessment.clinical_use_reason,
+ clinical_use_class: assessment.clinical_use_class,
+ clinical_use_reason: assessment.clinical_use_reason,
+ clinical_signal_score: assessment.clinical_signal_score,
+ admin_signal_score: assessment.admin_signal_score,
};
} catch {
+ const assessment = assessClinicalImageUse({
+ imageType: "unclear",
+ searchable: true,
+ clinicalRelevanceScore: 0.4,
+ sourceKind: args.sourceKind,
+ tableRole: args.tableRole,
+ tableText: args.tableText,
+ tableTitle: args.tableTitle,
+ tableLabel: args.tableLabel,
+ caption: response.text,
+ });
return {
image_type: "unclear" as const,
- searchable: true,
- clinical_relevance_score: 0.4,
+ searchable: assessment.searchable,
+ clinical_relevance_score: assessment.clinical_relevance_score,
labels: [],
caption: response.text.trim() || "Extracted source image.",
- skip_reason: null,
+ skip_reason: assessment.searchable ? null : assessment.clinical_use_reason,
+ clinical_use_class: assessment.clinical_use_class,
+ clinical_use_reason: assessment.clinical_use_reason,
+ clinical_signal_score: assessment.clinical_signal_score,
+ admin_signal_score: assessment.admin_signal_score,
};
}
}
diff --git a/src/lib/public-rate-limit.ts b/src/lib/public-rate-limit.ts
new file mode 100644
index 000000000..076943081
--- /dev/null
+++ b/src/lib/public-rate-limit.ts
@@ -0,0 +1,96 @@
+type RateLimitBucket = {
+ count: number;
+ resetAt: number;
+};
+
+export type PublicRateLimitResult = {
+ limited: boolean;
+ limit: number;
+ remaining: number;
+ retryAfterSeconds: number;
+ resetAt: number;
+};
+
+const answerBuckets = new Map();
+const searchBuckets = new Map();
+
+export const publicAnswerRateLimitDefaults = {
+ limit: 30,
+ windowMs: 60_000,
+} as const;
+
+export const publicSearchRateLimitDefaults = {
+ limit: 240,
+ windowMs: 60_000,
+} as const;
+
+export function publicRateLimitKey(headers: Headers) {
+ const forwardedFor = headers.get("x-forwarded-for")?.split(",")[0]?.trim();
+ if (forwardedFor) return forwardedFor;
+
+ const realIp = headers.get("x-real-ip")?.trim();
+ return realIp || "unknown";
+}
+
+function consumePublicRateLimit(
+ buckets: Map,
+ headers: Headers,
+ now = Date.now(),
+ options: { limit: number; windowMs: number },
+): PublicRateLimitResult {
+ const limit = options.limit;
+ const windowMs = options.windowMs;
+ const key = publicRateLimitKey(headers);
+ const existing = buckets.get(key);
+ const bucket =
+ existing && existing.resetAt > now
+ ? existing
+ : {
+ count: 0,
+ resetAt: now + windowMs,
+ };
+
+ bucket.count += 1;
+ buckets.set(key, bucket);
+
+ const retryAfterSeconds = Math.max(1, Math.ceil((bucket.resetAt - now) / 1000));
+ const remaining = Math.max(0, limit - bucket.count);
+
+ return {
+ limited: bucket.count > limit,
+ limit,
+ remaining,
+ retryAfterSeconds,
+ resetAt: bucket.resetAt,
+ };
+}
+
+export function consumePublicAnswerRateLimit(
+ headers: Headers,
+ now = Date.now(),
+ options: { limit?: number; windowMs?: number } = {},
+): PublicRateLimitResult {
+ return consumePublicRateLimit(answerBuckets, headers, now, {
+ limit: options.limit ?? publicAnswerRateLimitDefaults.limit,
+ windowMs: options.windowMs ?? publicAnswerRateLimitDefaults.windowMs,
+ });
+}
+
+export function consumePublicSearchRateLimit(
+ headers: Headers,
+ now = Date.now(),
+ options: { limit?: number; windowMs?: number } = {},
+): PublicRateLimitResult {
+ return consumePublicRateLimit(searchBuckets, headers, now, {
+ limit: options.limit ?? publicSearchRateLimitDefaults.limit,
+ windowMs: options.windowMs ?? publicSearchRateLimitDefaults.windowMs,
+ });
+}
+
+export function resetPublicAnswerRateLimitForTests() {
+ answerBuckets.clear();
+}
+
+export function resetPublicSearchRateLimitForTests() {
+ searchBuckets.clear();
+}
diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts
index ccf1d67ea..9791acea1 100644
--- a/src/lib/rag-eval-cases.ts
+++ b/src/lib/rag-eval-cases.ts
@@ -175,6 +175,46 @@ export const ragEvalCases: RagEvalCase[] = [
minCitations: 2,
latencyTargetMs: 2000,
},
+ {
+ id: "direct-document-lookup-nocc",
+ question: "Find the NOCC document",
+ category: "routine",
+ supported: true,
+ expectedFiles: ["MHSP.NOCC.pdf"],
+ allowedRoutes: ["extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "agitation-arousal-table-lookup",
+ question: "Which table covers agitation and arousal pharmacological management?",
+ category: "routine",
+ supported: true,
+ expectedFiles: ["MHSP.AgitationArousalPharmaMgt.pdf"],
+ allowedRoutes: ["extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "clozapine-fbc-acronym-threshold",
+ question: "What FBC threshold should withhold clozapine?",
+ category: "complex",
+ supported: true,
+ expectedFiles: ["CG.MHSP.ClozapinePresAdminMonitor.pdf"],
+ allowedRoutes: ["strong", "extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 20000,
+ },
+ {
+ id: "admission-discharge-comparison",
+ question: "Compare admission and discharge requirements",
+ category: "complex",
+ supported: true,
+ expectedFiles: ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+ allowedRoutes: ["strong", "extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 20000,
+ },
{
id: "neuroleptic-side-effect-escalation",
question: "When should neuroleptic side effects be escalated?",
@@ -195,6 +235,57 @@ export const ragEvalCases: RagEvalCase[] = [
minCitations: 2,
latencyTargetMs: 20000,
},
+ {
+ id: "clozapine-monitoring-paraphrase",
+ question: "Which observations and blood monitoring are needed while a patient is taking clozapine?",
+ category: "routine",
+ supported: true,
+ expectedFiles: ["CG.MHSP.ClozapinePresAdminMonitor.pdf"],
+ allowedRoutes: ["extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "clozapine-typo-acronym-threshold",
+ question: "What ANC or FBC cut off means clozapin should be withheld?",
+ category: "complex",
+ supported: true,
+ expectedFiles: ["CG.MHSP.ClozapinePresAdminMonitor.pdf"],
+ allowedRoutes: ["strong", "extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 20000,
+ },
+ {
+ id: "clozapine-missed-dose-table",
+ question: "Show the clozapine missed-dose monitoring table guidance.",
+ category: "routine",
+ supported: true,
+ expectedFiles: ["CG.MHSP.ClozapinePresAdminMonitor.pdf"],
+ allowedRoutes: ["extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 2000,
+ requireVisualEvidence: true,
+ },
+ {
+ id: "agitation-arousal-typo-dosing",
+ question: "What agitaton and arousl dosing guidance applies to psychiatric inpatients?",
+ category: "routine",
+ supported: true,
+ expectedFiles: ["MHSP.AgitationArousalPharmaMgt.pdf"],
+ allowedRoutes: ["extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "admission-discharge-coverage-paraphrase",
+ question: "Combine community admission steps with discharge documentation requirements.",
+ category: "complex",
+ supported: true,
+ expectedFiles: ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+ allowedRoutes: ["strong", "extractive", "fast"],
+ minCitations: 1,
+ latencyTargetMs: 20000,
+ },
{
id: "unsupported-dka-insulin",
question: "What is the diabetic ketoacidosis insulin protocol?",
@@ -215,6 +306,36 @@ export const ragEvalCases: RagEvalCase[] = [
minCitations: 0,
latencyTargetMs: 2000,
},
+ {
+ id: "unsupported-ssri-adolescent-dose",
+ question: "What SSRI dose is recommended for adolescent depression?",
+ category: "unsupported",
+ supported: false,
+ expectedFiles: [],
+ allowedRoutes: ["unsupported"],
+ minCitations: 0,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "unsupported-hyperkalaemia-insulin",
+ question: "What insulin dose should be used for hyperkalaemia?",
+ category: "unsupported",
+ supported: false,
+ expectedFiles: [],
+ allowedRoutes: ["unsupported"],
+ minCitations: 0,
+ latencyTargetMs: 2000,
+ },
+ {
+ id: "unsupported-future-upload-title",
+ question: "Find the newly uploaded Future Synthetic Ketamine Sedation Protocol.",
+ category: "unsupported",
+ supported: false,
+ expectedFiles: [],
+ allowedRoutes: ["unsupported"],
+ minCitations: 0,
+ latencyTargetMs: 2000,
+ },
];
export function selectRagEvalCases(args: { limit?: number; question?: string }) {
diff --git a/src/lib/rag-routing.ts b/src/lib/rag-routing.ts
index 5b2a5e2c1..e3aa88694 100644
--- a/src/lib/rag-routing.ts
+++ b/src/lib/rag-routing.ts
@@ -1,4 +1,5 @@
-import type { ConflictOrGap, RagAnswer, SearchResult } from "@/lib/types";
+import { classifyRagQuery } from "@/lib/clinical-search";
+import type { ConflictOrGap, RagAnswer, RagQueryClass, SearchResult } from "@/lib/types";
export type AnswerRouteMode = "unsupported" | "extractive" | "fast" | "strong";
@@ -15,7 +16,9 @@ const strongRetrievalThreshold = 0.64;
const extractiveRetrievalThreshold = 0.76;
const complexClinicalQueryPattern =
/\b(compare|compared|versus|vs|conflict|gap|contraindicat\w*|interaction\w*|side effect\w*|adverse|suicid\w*|toxicity|myocarditis|neutropenia|anc|fbc|urgent|escalat\w*|withhold|cease|stop|dose|dosing|prescrib\w*)\b/i;
-const comparisonQueryPattern = /\b(compare|compared|versus|vs|between|across|difference\w*|conflict\w*)\b/i;
+const comparisonQueryPattern = /\b(compare|compared|versus|vs|between|difference\w*|conflict\w*)\b/i;
+const routineCrossDocumentPattern =
+ /\b(?:across|combine|combined|synthesi[sz]e|together|overall|all documents|these documents|different documents|multiple documents|several documents|from the documents)\b/i;
const extractiveQuestionPattern =
/\b(what|when|where|which|who|list|include|includes|required|requirements|process|procedure|steps|monitoring|summary|summarise|summarize|show|tell)\b/i;
const extractiveBlockPattern =
@@ -56,8 +59,18 @@ function hasTextSupport(results: SearchResult[]) {
function hasActionableConflictOrGap(conflictsOrGaps: ConflictOrGap[] = []) {
return conflictsOrGaps.some(
(item) =>
- item.type === "conflict" ||
- /limited-strength|not enough|no indexed|weak support|unsupported/i.test(item.message),
+ item.type === "conflict" || /limited-strength|not enough|no indexed|weak support|unsupported/i.test(item.message),
+ );
+}
+
+function hasConflictIntent(query: string) {
+ return /\b(?:conflict|gap|contradict|disagree|inconsisten|versus|vs)\b/i.test(query);
+}
+
+function hasExplicitDocumentLookupIntent(query: string) {
+ return (
+ /\b(?:find|search|lookup|open|show)\b.{0,80}\b(?:document|file|pdf|protocol|guideline|procedure)\b/i.test(query) ||
+ /\bnewly uploaded\b/i.test(query)
);
}
@@ -93,6 +106,7 @@ export function isComplexClinicalQuery(query: string) {
export function shouldUseExtractiveAnswer(args: {
query: string;
results: SearchResult[];
+ queryClass?: RagQueryClass;
conflictsOrGaps?: ConflictOrGap[];
}) {
if (args.results.length === 0) return false;
@@ -100,19 +114,40 @@ export function shouldUseExtractiveAnswer(args: {
const strongestScore = strongestRetrievalScore(args.results);
const topTextRank = Math.max(...args.results.map((result) => result.text_rank ?? 0));
const documents = documentCount(args.results);
+ const queryClass = args.queryClass ?? classifyRagQuery(args.query).queryClass;
+ const singleDocumentStrongMatch = documents === 1 && strongestScore >= 0.74 && (topTextRank >= 0.04 || hasTextSupport(args.results));
if (documents > 1 && comparisonQueryPattern.test(args.query)) return false;
+ if (queryClass === "comparison" || queryClass === "broad_summary") return false;
if (extractiveBlockPattern.test(args.query) && !directTitleSupport) return false;
if (!extractiveQuestionPattern.test(args.query) && !directTitleSupport) return false;
if (hasActionableConflictOrGap(args.conflictsOrGaps) && !directTitleSupport && strongestScore < 0.82) return false;
- return strongestScore >= extractiveRetrievalThreshold || topTextRank >= 0.12 || (directTitleSupport && strongestScore >= 0.4);
+ if (queryClass === "table_threshold" && strongestScore >= 0.68 && (topTextRank >= 0.035 || singleDocumentStrongMatch)) {
+ return true;
+ }
+
+ if (queryClass === "document_lookup" && (directTitleSupport || strongestScore >= 0.72)) {
+ return true;
+ }
+
+ if (queryClass === "medication_dose_risk" && singleDocumentStrongMatch && !comparisonQueryPattern.test(args.query)) {
+ return true;
+ }
+
+ return (
+ strongestScore >= extractiveRetrievalThreshold ||
+ singleDocumentStrongMatch ||
+ topTextRank >= 0.12 ||
+ (directTitleSupport && strongestScore >= 0.4)
+ );
}
export function chooseAnswerRoute(args: {
query: string;
results: SearchResult[];
+ queryClass?: RagQueryClass;
conflictsOrGaps?: ConflictOrGap[];
fastModel: string;
strongModel: string;
@@ -120,6 +155,8 @@ export function chooseAnswerRoute(args: {
const strongestScore = strongestRetrievalScore(args.results);
const documents = documentCount(args.results);
const directTitleSupport = hasDirectTitleSupport(args.query, args.results);
+ const queryClass = args.queryClass ?? classifyRagQuery(args.query).queryClass;
+ const topTextRank = Math.max(0, ...args.results.map((result) => result.text_rank ?? 0));
if (args.results.length === 0) {
return {
@@ -141,47 +178,119 @@ export function chooseAnswerRoute(args: {
};
}
- if (shouldUseExtractiveAnswer(args)) {
+ if (
+ queryClass === "document_lookup" &&
+ hasExplicitDocumentLookupIntent(args.query) &&
+ !directTitleSupport &&
+ topTextRank < 0.08
+ ) {
return {
- mode: "extractive",
+ mode: "unsupported",
model: null,
- reason: "strong_source_match_extract",
+ reason: "document_lookup_without_title_support",
strongestScore,
documentCount: documents,
};
}
- if (isComplexClinicalQuery(args.query)) {
+ if (
+ (queryClass === "medication_dose_risk" || queryClass === "table_threshold") &&
+ strongestScore < 0.46 &&
+ topTextRank < 0.02 &&
+ !directTitleSupport
+ ) {
+ return {
+ mode: "unsupported",
+ model: null,
+ reason: "weak_complex_query_support",
+ strongestScore,
+ documentCount: documents,
+ };
+ }
+
+ const crossDocumentIntent = routineCrossDocumentPattern.test(args.query) || queryClass === "broad_summary";
+ const actionableConflictOrGap = hasActionableConflictOrGap(args.conflictsOrGaps);
+
+ if (
+ documents > 1 &&
+ (queryClass === "comparison" || comparisonQueryPattern.test(args.query)) &&
+ documents <= 3 &&
+ strongestScore >= 0.72 &&
+ !hasConflictIntent(args.query) &&
+ !actionableConflictOrGap
+ ) {
+ return {
+ mode: "fast",
+ model: args.fastModel,
+ reason: "balanced_multi_document_synthesis",
+ strongestScore,
+ documentCount: documents,
+ };
+ }
+
+ if (
+ documents > 1 &&
+ crossDocumentIntent &&
+ strongestScore >= strongRetrievalThreshold &&
+ !hasConflictIntent(args.query) &&
+ !actionableConflictOrGap
+ ) {
+ return {
+ mode: "fast",
+ model: args.fastModel,
+ reason: "balanced_multi_document_synthesis",
+ strongestScore,
+ documentCount: documents,
+ };
+ }
+
+ if (queryClass === "comparison" || (documents > 3 && comparisonQueryPattern.test(args.query) && !directTitleSupport)) {
return {
mode: "strong",
model: args.strongModel,
- reason: "clinical_risk_or_complex_query",
+ reason: "multi_document_comparison_synthesis",
strongestScore,
documentCount: documents,
};
}
- if (strongestScore < strongRetrievalThreshold && !directTitleSupport) {
+ if (shouldUseExtractiveAnswer({ ...args, queryClass })) {
+ const reason =
+ queryClass === "table_threshold"
+ ? "table_or_threshold_source_extractive"
+ : queryClass === "document_lookup"
+ ? "document_lookup_source_extractive"
+ : "high_confidence_source_extractive";
+ return {
+ mode: "extractive",
+ model: null,
+ reason,
+ strongestScore,
+ documentCount: documents,
+ };
+ }
+
+ if (isComplexClinicalQuery(args.query)) {
return {
mode: "strong",
model: args.strongModel,
- reason: "limited_retrieval_strength",
+ reason: "clinical_risk_or_complex_query",
strongestScore,
documentCount: documents,
};
}
- if (documents > 3 && comparisonQueryPattern.test(args.query) && !directTitleSupport) {
+ if (strongestScore < strongRetrievalThreshold && !directTitleSupport) {
return {
mode: "strong",
model: args.strongModel,
- reason: "multi_document_synthesis",
+ reason: "limited_retrieval_strength",
strongestScore,
documentCount: documents,
};
}
- if (hasActionableConflictOrGap(args.conflictsOrGaps) && !directTitleSupport) {
+ if (actionableConflictOrGap && !directTitleSupport) {
return {
mode: "strong",
model: args.strongModel,
diff --git a/src/lib/rag.ts b/src/lib/rag.ts
index b0ebea843..a2146bd50 100644
--- a/src/lib/rag.ts
+++ b/src/lib/rag.ts
@@ -1,11 +1,32 @@
import { createAdminClient } from "@/lib/supabase/admin";
import { embedTextWithTelemetry, generateStructuredTextResult, type OpenAITextResult } from "@/lib/openai";
import { compactCitations } from "@/lib/citations";
-import { buildClinicalTextSearchQuery, expandClinicalQuery, rankClinicalResults } from "@/lib/clinical-search";
+import {
+ buildClinicalTextSearchQuery,
+ classifyRagQuery,
+ expandClinicalQuery,
+ hasDoseEvidenceSupport,
+ rankClinicalResults,
+} from "@/lib/clinical-search";
import { env } from "@/lib/env";
import { normalizeSourceMetadata } from "@/lib/source-metadata";
+import { isClinicalImageEvidence } from "@/lib/image-filtering";
import { chooseAnswerRoute, hasDirectTitleSupport, shouldRetryWithStrongAfterFast } from "@/lib/rag-routing";
-import { fetchRelatedDocuments } from "@/lib/document-enrichment";
+import { fetchRelatedDocumentMetadata, fetchRelatedDocuments } from "@/lib/document-enrichment";
+import {
+ boldHighYieldClinicalText,
+ boldRagAnswerHighYieldText,
+ rankAnswerEvidence,
+} from "@/lib/answer-ranking";
+import { applyMemoryCardBoosts, fetchMemoryCardsForQuery, ragDeepMemoryVersion } from "@/lib/deep-memory";
+import { cleanClinicalSummaryText, sourceTextForDisplay, sourceTextForModel } from "@/lib/source-text-sanitizer";
+import {
+ buildCrossDocumentFusionBrief,
+ buildCrossDocumentSourceGuide,
+ buildCrossDocumentSynthesisPlan,
+} from "@/lib/cross-document-synthesis";
+import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
+import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance";
import { z } from "zod";
import {
buildDocumentBreakdown,
@@ -19,7 +40,23 @@ import {
reconcileQuoteCards,
selectBestSourceRecommendation,
} from "@/lib/evidence";
-import type { AnswerSection, Citation, ConflictOrGap, OpenAITokenUsage, QuoteCard, RagAnswer, SearchResult } from "@/lib/types";
+import type {
+ AnswerSection,
+ ChunkImage,
+ ClinicalImageUseClass,
+ Citation,
+ ConflictOrGap,
+ DocumentIndexQuality,
+ DocumentMemoryCard,
+ EvidenceRelevance,
+ RelatedDocument,
+ OpenAITokenUsage,
+ QuoteCard,
+ RagQueryClass,
+ RagAnswer,
+ SearchResult,
+ SmartRagApiPlan,
+} from "@/lib/types";
const answerJsonOutputSchema = {
type: "object",
@@ -29,6 +66,7 @@ const answerJsonOutputSchema = {
answer: {
type: "string",
description: "The concise answer or a clear statement that the provided excerpts are insufficient.",
+ maxLength: 1200,
},
grounded: {
type: "boolean",
@@ -42,12 +80,13 @@ const answerJsonOutputSchema = {
answerSections: {
type: "array",
description: "Optional clinically useful sections. Omit unsupported detail by returning an empty array.",
+ maxItems: 3,
items: {
type: "object",
additionalProperties: false,
properties: {
- heading: { type: "string", description: "Short section heading." },
- body: { type: "string", description: "Section body grounded in the cited excerpts." },
+ heading: { type: "string", description: "Short section heading.", maxLength: 48 },
+ body: { type: "string", description: "Section body grounded in the cited excerpts.", maxLength: 420 },
citation_chunk_ids: {
type: "array",
description: "Retrieved chunk IDs that directly support this section.",
@@ -60,6 +99,7 @@ const answerJsonOutputSchema = {
citations: {
type: "array",
description: "The strongest retrieved chunk IDs that directly support the answer.",
+ maxItems: 5,
items: {
type: "object",
additionalProperties: false,
@@ -72,12 +112,13 @@ const answerJsonOutputSchema = {
quoteCards: {
type: "array",
description: "Short exact quotes copied from supplied excerpts. Use an empty array if no exact quote is useful.",
+ maxItems: 3,
items: {
type: "object",
additionalProperties: false,
properties: {
chunk_id: { type: "string", description: "A citation_chunk_id from the supplied source block." },
- quote: { type: "string", description: "A short exact quote from the cited source excerpt." },
+ quote: { type: "string", description: "A short exact quote from the cited source excerpt.", maxLength: 260 },
section_heading: { type: ["string", "null"], description: "Source section heading when visible." },
},
required: ["chunk_id", "quote", "section_heading"],
@@ -86,11 +127,16 @@ const answerJsonOutputSchema = {
conflictsOrGaps: {
type: "array",
description: "Important gaps or conflicts found in the retrieved excerpts.",
+ maxItems: 3,
items: {
type: "object",
additionalProperties: false,
properties: {
- type: { type: "string", enum: ["gap", "conflict"], description: "Whether this is missing support or conflicting support." },
+ type: {
+ type: "string",
+ enum: ["gap", "conflict"],
+ description: "Whether this is missing support or conflicting support.",
+ },
message: { type: "string", description: "Plain-language gap or conflict statement." },
source_chunk_ids: {
type: "array",
@@ -112,6 +158,127 @@ const confidenceOrder = {
high: 3,
} as const;
+export const machineReadableFallbackAnswer =
+ "The indexed sources were not machine-readable enough to produce a formatted answer.";
+
+function safeRecord(value: unknown) {
+ return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {};
+}
+
+function metadataText(metadata: Record, key: string) {
+ const value = metadata[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+const likelyFragmentPhrases =
+ /\b(?:answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)\b/i;
+const answerSectionArtifactPattern =
+ /"?(answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)"?\s*:\s*/i;
+
+function normalizeSectionText(value: string) {
+ return value.trim().replace(/\s+/g, " ");
+}
+
+function splitBalancedWords(text: string) {
+ return text
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .trim()
+ .split(/\s+/)
+ .filter(Boolean);
+}
+
+function looksLikeJsonArtifact(value: string) {
+ const normalized = normalizeSectionText(value);
+ if (!normalized) return true;
+ const tokenCount = normalized.split(/\s+/).filter(Boolean).length;
+ const hasJsonStructure = /[{}\[\]]/.test(normalized);
+ const quoteCount = (normalized.match(/"/g) ?? []).length;
+ const colonCount = (normalized.match(/:/g) ?? []).length;
+ const keyValuePairs = (normalized.match(/"[^"]+"\s*:\s*/g) ?? []).length;
+ const keyPairDensity = tokenCount > 0 ? keyValuePairs / tokenCount : 0;
+ const hasKnownKeys = likelyFragmentPhrases.test(normalized);
+ const hasBalancedBraces = (normalized.match(/[{}\[\]]/g) ?? []).length >= 2;
+ const hasBalancedBrackets = (normalized.match(/[\[\]]/g) ?? []).length >= 2;
+ const tokenDensity = splitBalancedWords(normalized);
+ const isMostlyPunctuationNoise = tokenDensity.length >= 6 && tokenDensity.every((word) => word.length <= 2);
+ const hasBracketKeyPairs =
+ /"\s*(?:answer|heading|body|grounded|confidence|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)\s*"/i.test(
+ normalized,
+ );
+
+ if (normalized.startsWith("{") && normalized.endsWith("}") && (hasKnownKeys || quoteCount >= 4)) {
+ return true;
+ }
+
+ if (
+ hasJsonStructure &&
+ hasBalancedBraces &&
+ keyValuePairs >= 2 &&
+ quoteCount >= 4 &&
+ colonCount >= 2 &&
+ hasKnownKeys &&
+ (tokenCount <= 70 || keyPairDensity > 0.2)
+ ) {
+ return true;
+ }
+
+ if (
+ normalized.includes("}") &&
+ normalized.includes("{") &&
+ hasKnownKeys &&
+ /"[^"]+":\s*"/.test(normalized) &&
+ tokenCount <= 40
+ ) {
+ return true;
+ }
+
+ if (isMostlyPunctuationNoise) return true;
+ if (
+ hasBracketKeyPairs &&
+ hasJsonStructure &&
+ (quoteCount >= 2 || colonCount >= 2 || hasBalancedBrackets) &&
+ tokenCount <= 70
+ ) {
+ return true;
+ }
+
+ return false;
+}
+
+function sanitizeStructuredText(
+ value: string,
+ options: { minLength?: number; minTokens?: number; keepLeading?: boolean } = {},
+) {
+ const { minLength = 2, minTokens = 1, keepLeading = false } = options;
+ const normalized = normalizeSectionText(sourceTextForDisplay(value));
+ if (!normalized) return "";
+
+ const trimmed =
+ normalized.search(answerSectionArtifactPattern) === 0
+ ? normalized.replace(answerSectionArtifactPattern, "").trim()
+ : normalized.search(answerSectionArtifactPattern) > 0
+ ? normalized.slice(0, normalized.search(answerSectionArtifactPattern)).trim()
+ : normalized;
+
+ const finalText = keepLeading ? trimmed : trimmed.trim();
+ if (!finalText) return "";
+ if (finalText.length < minLength) return "";
+ if (looksLikeJsonArtifact(finalText)) return "";
+ const tokenCount = finalText.split(/\s+/).filter(Boolean).length;
+ if (tokenCount < minTokens) return "";
+ if (!/[A-Za-z]{2,}/.test(finalText)) return "";
+ return finalText;
+}
+
+function sanitizeAnswerText(value: string) {
+ return sanitizeStructuredText(value, { minLength: 8, minTokens: 2, keepLeading: true });
+}
+
+function isUsableAnswerSectionText(value: string, options: { minTokens?: number; minLength?: number } = {}) {
+ return Boolean(sanitizeStructuredText(value, options));
+}
+
export type SearchChunksArgs = {
query: string;
topK?: number;
@@ -126,22 +293,40 @@ export type AnswerProgressEvent = {
stage: "retrieved" | "routing" | "generating" | "retrying" | "finalizing" | "cached";
message: string;
resultCount?: number;
+ visibleSourceCount?: number;
+ directSourceCount?: number;
+ weakSourceCount?: number;
+ timingMs?: number;
+ relevance?: EvidenceRelevance;
mode?: RagAnswer["routingMode"];
model?: string | null;
reason?: string;
+ smartApiPlan?: SmartRagApiPlan;
};
export type SearchTelemetry = {
search_cache_hit: boolean;
+ query_class?: RagQueryClass;
text_fast_path_latency_ms: number;
embedding_skipped: boolean;
embedding_latency_ms: number;
embedding_cache_hit: boolean;
supabase_rpc_latency_ms: number;
rerank_latency_ms: number;
- retrieval_strategy?: "search_cache" | "text_fast_path" | "hybrid" | "vector_fallback";
+ memory_card_count?: number;
+ memory_top_score?: number;
+ weighted_top_score?: number;
+ rrf_top_score?: number;
+ retrieval_strategy?: "search_cache" | "text_fast_path" | "document_lookup_fast_path" | "hybrid" | "vector_fallback";
};
+function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) {
+ telemetry.weighted_top_score = Number(
+ Math.max(0, ...results.map((result) => result.hybrid_score ?? result.similarity ?? 0)).toFixed(4),
+ );
+ telemetry.rrf_top_score = Number(Math.max(0, ...results.map((result) => result.rrf_score ?? 0)).toFixed(4));
+}
+
const citationSchema = z.object({
chunk_id: z.string(),
document_id: z.string().optional(),
@@ -236,15 +421,29 @@ function sanitizeCitations(proposed: Array<{ chunk_id: string }> | undefined, re
return compactCitations(results);
}
-function sanitizeAnswerSections(sections: AnswerSection[] | undefined, results: SearchResult[]): AnswerSection[] {
+function sanitizeAnswerSections(
+ sections: AnswerSection[] | undefined,
+ results: SearchResult[],
+ query?: string,
+): AnswerSection[] {
const allowed = new Set(results.map((result) => result.id));
+ const seen = new Set();
+
return (sections ?? [])
.map((section) => ({
- heading: section.heading,
- body: section.body,
- citation_chunk_ids: section.citation_chunk_ids.filter((id) => allowed.has(id)),
+ heading: sanitizeStructuredText(section.heading, { minLength: 1, minTokens: 1 }),
+ body: boldHighYieldClinicalText(sanitizeStructuredText(section.body, { minLength: 8, minTokens: 2 }), query),
+ citation_chunk_ids: [...new Set(section.citation_chunk_ids.filter((id) => allowed.has(id)))],
}))
- .filter((section) => section.citation_chunk_ids.length > 0);
+ .filter((section) => {
+ if (!section.heading || !section.body || section.citation_chunk_ids.length === 0) return false;
+ if (!isUsableAnswerSectionText(section.heading, { minTokens: 1, minLength: 1 })) return false;
+ if (!isUsableAnswerSectionText(section.body, { minTokens: 2, minLength: 8 })) return false;
+ const key = `${section.heading.toLowerCase()}||${section.body.toLowerCase()}`;
+ if (seen.has(key)) return false;
+ seen.add(key);
+ return true;
+ });
}
function sanitizeQuoteCards(
@@ -256,9 +455,11 @@ function sanitizeQuoteCards(
.map((card) => {
const source = chunks.get(card.chunk_id);
if (!source) return null;
+ const quote = sanitizeStructuredText(card.quote, { minLength: 8, minTokens: 2 });
+ if (!quote) return null;
return {
...resultCitation(source),
- quote: card.quote,
+ quote,
section_heading: card.section_heading ?? source.section_heading,
} satisfies QuoteCard;
})
@@ -270,7 +471,7 @@ function sanitizeConflictsOrGaps(items: ConflictOrGap[] | undefined, results: Se
return (items ?? [])
.map((item) => ({
type: item.type,
- message: item.message,
+ message: sanitizeStructuredText(item.message, { minLength: 8, minTokens: 2 }) || item.message,
source_chunk_ids: item.source_chunk_ids?.filter((id) => allowed.has(id)),
}))
.filter((item) => !item.source_chunk_ids || item.source_chunk_ids.length > 0);
@@ -283,15 +484,16 @@ function normalizeSearchResults(results: SearchResult[]) {
}));
}
-function safeFallbackAnswer(raw: string, results: SearchResult[]): RagAnswer {
+function safeFallbackAnswer(raw: string, results: SearchResult[], query?: string): RagAnswer {
const citations = compactCitations(results);
const confidence = deriveConfidence(results, citations.length);
return {
- answer: raw || "The model returned an invalid response. Verify the retrieved sources before using this answer.",
+ answer: boldHighYieldClinicalText(sanitizeAnswerText(raw) || machineReadableFallbackAnswer, query),
grounded: citations.length > 0 && confidence !== "unsupported",
confidence,
citations,
sources: results,
+ routingReason: "structured_parse_fallback",
answerSections: [],
conflictsOrGaps: detectConflictsOrGaps(results),
visualEvidence: buildVisualEvidence(results),
@@ -314,6 +516,16 @@ function hasOpenAIUsage(usage: OpenAITokenUsage) {
return Object.values(usage).some((value) => typeof value === "number" && value > 0);
}
+function fallbackReasonFromRouting(reason?: string | null) {
+ if (!reason) return null;
+ return (
+ reason
+ .split(";")
+ .map((part) => part.trim())
+ .find((part) => /fallback|unsupported|no_|limited_retrieval|gap|conflict|failed/i.test(part)) ?? null
+ );
+}
+
const answerCache = new Map();
const searchCache = new Map();
@@ -381,13 +593,7 @@ function scopedSearchCacheKey(args: SearchChunksArgs) {
? args.documentId
: "all-documents";
const normalizedQuery = buildClinicalTextSearchQuery(args.query).toLowerCase().replace(/\s+/g, " ");
- return [
- args.ownerId ?? "anonymous",
- scope,
- normalizedQuery,
- args.topK ?? 8,
- args.minSimilarity ?? 0.15,
- ].join("|");
+ return [args.ownerId ?? "anonymous", scope, normalizedQuery, args.topK ?? 8, args.minSimilarity ?? 0.15].join("|");
}
function cloneSearchResults(results: SearchResult[]) {
@@ -436,6 +642,22 @@ function setCachedSearch(args: SearchChunksArgs, results: SearchResult[], teleme
}
}
+export function invalidateRagCachesForOwner(ownerId?: string | null) {
+ if (!ownerId) {
+ answerCache.clear();
+ searchCache.clear();
+ return;
+ }
+
+ const prefix = `${ownerId}|`;
+ for (const key of answerCache.keys()) {
+ if (key.startsWith(prefix)) answerCache.delete(key);
+ }
+ for (const key of searchCache.keys()) {
+ if (key.startsWith(prefix)) searchCache.delete(key);
+ }
+}
+
async function insertRagQuery(row: Record) {
const supabase = createAdminClient();
await supabase.from("rag_queries").insert(row);
@@ -468,23 +690,736 @@ function mergeSearchResults(primary: SearchResult[], secondary: SearchResult[])
return Array.from(merged.values());
}
+type DocumentLookupRow = {
+ id: string;
+ owner_id?: string | null;
+ title: string;
+ file_name: string;
+ status?: string;
+ page_count?: number;
+ chunk_count?: number;
+ image_count?: number;
+ metadata?: unknown;
+ text_rank?: number;
+ match_reason?: string;
+};
+
+type DocumentLookupChunkRow = {
+ id: string;
+ document_id: string;
+ page_number: number | null;
+ chunk_index: number;
+ section_heading: string | null;
+ content: string;
+ image_ids: string[] | null;
+};
+
+async function searchDocumentLookupFastPath(args: {
+ supabase: ReturnType;
+ query: string;
+ ownerId?: string;
+ documentIds?: string[];
+ matchCount: number;
+}) {
+ const { data: documents, error: documentsError } = await args.supabase.rpc("match_documents_for_query", {
+ query_text: buildClinicalTextSearchQuery(args.query),
+ match_count: 12,
+ owner_filter: args.ownerId ?? null,
+ });
+ if (documentsError || !documents?.length) return [];
+
+ const allowedDocuments = args.documentIds?.length ? new Set(args.documentIds) : null;
+ const rankedDocuments = (documents as DocumentLookupRow[])
+ .filter((document) => !allowedDocuments || allowedDocuments.has(document.id))
+ .map((document) => ({
+ document,
+ score: Math.min(0.34, Number(document.text_rank ?? 0)),
+ }))
+ .filter((item) => item.score > 0)
+ .sort((a, b) => b.score - a.score)
+ .slice(0, 5);
+
+ if (rankedDocuments.length === 0) return [];
+
+ const documentById = new Map(rankedDocuments.map((item) => [item.document.id, item.document]));
+ const scoreByDocument = new Map(rankedDocuments.map((item) => [item.document.id, item.score]));
+ const { data: chunks, error: chunksError } = await args.supabase
+ .from("document_chunks")
+ .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids")
+ .in(
+ "document_id",
+ rankedDocuments.map((item) => item.document.id),
+ )
+ .order("chunk_index", { ascending: true })
+ .limit(Math.max(args.matchCount, rankedDocuments.length * 4));
+
+ if (chunksError || !chunks?.length) return [];
+
+ const results: SearchResult[] = [];
+ for (const chunk of chunks as DocumentLookupChunkRow[]) {
+ const document = documentById.get(chunk.document_id);
+ if (!document) continue;
+ const documentScore = scoreByDocument.get(chunk.document_id) ?? 0;
+ const similarity = Math.min(0.92, 0.6 + documentScore);
+ results.push({
+ id: chunk.id,
+ document_id: chunk.document_id,
+ title: document.title,
+ file_name: document.file_name,
+ page_number: chunk.page_number,
+ chunk_index: chunk.chunk_index,
+ section_heading: chunk.section_heading,
+ content: chunk.content,
+ image_ids: chunk.image_ids ?? [],
+ source_metadata: normalizeSourceMetadata(document.metadata),
+ similarity,
+ text_rank: documentScore,
+ hybrid_score: Math.min(0.94, similarity + 0.02),
+ images: [],
+ });
+ }
+
+ return results
+ .sort((a, b) => (b.hybrid_score ?? b.similarity) - (a.hybrid_score ?? a.similarity) || a.chunk_index - b.chunk_index)
+ .slice(0, args.matchCount);
+}
+
+function collectMemoryCards(results: SearchResult[], limit = 8) {
+ const seen = new Set();
+ const cards: DocumentMemoryCard[] = [];
+ for (const result of results) {
+ for (const card of result.memory_cards ?? []) {
+ const key = card.id ?? `${card.document_id}:${card.card_type}:${card.title}:${card.content}`;
+ if (seen.has(key)) continue;
+ seen.add(key);
+ cards.push(card);
+ if (cards.length >= limit) return cards;
+ }
+ }
+ return cards;
+}
+
+function buildIndexingQuality(results: SearchResult[], memoryCards: DocumentMemoryCard[]): DocumentIndexQuality {
+ const sourceMetadata = results.map((result) => normalizeSourceMetadata(result.source_metadata));
+ const extractionQuality = sourceMetadata.some((metadata) => metadata.extraction_quality === "poor")
+ ? "poor"
+ : sourceMetadata.some((metadata) => metadata.extraction_quality === "partial")
+ ? "partial"
+ : sourceMetadata.length > 0
+ ? "good"
+ : "unknown";
+ return {
+ indexingVersion: ragDeepMemoryVersion,
+ memoryVersion: ragDeepMemoryVersion,
+ extractionQuality,
+ memoryCardCount: memoryCards.length,
+ stale: sourceMetadata.some((metadata) => metadata.document_status === "outdated"),
+ };
+}
+
+function buildAnswerScoreExplanations(results: SearchResult[], limit = 8): NonNullable {
+ return results.slice(0, limit).map((result) => ({
+ chunk_id: result.id,
+ document_id: result.document_id,
+ finalScore: Number((result.score_explanation?.finalScore ?? result.hybrid_score ?? result.similarity ?? 0).toFixed(4)),
+ score_explanation: result.score_explanation,
+ }));
+}
+
+function scoreExplanationLogMetadata(scoreExplanations: NonNullable) {
+ return {
+ score_explanation_count: scoreExplanations.length,
+ top_cited_score_explanations: scoreExplanations.slice(0, 8).map((entry) => ({
+ chunk_id: entry.chunk_id,
+ document_id: entry.document_id,
+ final_score: entry.finalScore,
+ vector_score: entry.score_explanation?.vectorScore ?? null,
+ text_rank: entry.score_explanation?.textRank ?? null,
+ weighted_hybrid_score: entry.score_explanation?.weightedHybridScore ?? null,
+ rrf_score: entry.score_explanation?.rrfScore ?? null,
+ memory_boost: entry.score_explanation?.memoryBoost ?? null,
+ title_boost: entry.score_explanation?.titleBoost ?? null,
+ metadata_boost: entry.score_explanation?.metadataBoost ?? null,
+ clinical_signal_boost: entry.score_explanation?.clinicalSignalBoost ?? null,
+ penalty: entry.score_explanation?.penalty ?? null,
+ final_rank: entry.score_explanation?.finalRank ?? null,
+ })),
+ };
+}
+
+function memoryCardChunkScore(card: DocumentMemoryCard) {
+ const hybridScore = Number(card.metadata?.memory_hybrid_score);
+ if (Number.isFinite(hybridScore) && hybridScore > 0) return Math.min(1, hybridScore);
+ return Math.min(1, card.confidence ?? 0.5);
+}
+
+async function loadChunksForMemoryCards(
+ supabase: ReturnType,
+ cards: DocumentMemoryCard[],
+ ownerId?: string,
+) {
+ const chunkIds = Array.from(new Set(cards.flatMap((card) => card.source_chunk_ids ?? []))).slice(0, 80);
+ if (chunkIds.length === 0) return [] as SearchResult[];
+
+ const { data: chunks, error: chunksError } = await supabase
+ .from("document_chunks")
+ .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids")
+ .in("id", chunkIds)
+ .limit(chunkIds.length);
+ if (chunksError || !chunks?.length) return [] as SearchResult[];
+
+ const documentIds = Array.from(new Set(chunks.map((chunk) => chunk.document_id)));
+ let documentQuery = supabase.from("documents").select("id,title,file_name,metadata,owner_id").in("id", documentIds);
+ if (ownerId) documentQuery = documentQuery.eq("owner_id", ownerId);
+ const { data: documents, error: documentsError } = await documentQuery;
+ if (documentsError || !documents?.length) return [] as SearchResult[];
+
+ const documentById = new Map(documents.map((document) => [document.id, document]));
+ const bestCardByChunk = new Map();
+ for (const card of cards) {
+ for (const chunkId of card.source_chunk_ids ?? []) {
+ const existing = bestCardByChunk.get(chunkId);
+ if (!existing || memoryCardChunkScore(card) > memoryCardChunkScore(existing)) bestCardByChunk.set(chunkId, card);
+ }
+ }
+
+ return chunks
+ .map((chunk) => {
+ const document = documentById.get(chunk.document_id);
+ if (!document) return null;
+ const card = bestCardByChunk.get(chunk.id);
+ const similarity = Math.min(0.92, 0.58 + (card?.confidence ?? 0.5) * 0.28);
+ return {
+ id: chunk.id,
+ document_id: chunk.document_id,
+ title: document.title,
+ file_name: document.file_name,
+ page_number: chunk.page_number,
+ chunk_index: chunk.chunk_index,
+ section_heading: chunk.section_heading,
+ content: chunk.content,
+ image_ids: chunk.image_ids ?? [],
+ source_metadata: normalizeSourceMetadata(document.metadata),
+ similarity,
+ text_rank: card?.confidence ?? 0,
+ hybrid_score: Math.min(0.96, similarity + 0.03),
+ images: [],
+ } satisfies SearchResult;
+ })
+ .filter(Boolean) as SearchResult[];
+}
+
+async function withMemoryBoostedCandidates(args: {
+ supabase: ReturnType;
+ query: string;
+ candidates: SearchResult[];
+ queryEmbedding?: number[];
+ ownerId?: string;
+ documentIds?: string[];
+ matchCount: number;
+}) {
+ const cards = await fetchMemoryCardsForQuery({
+ supabase: args.supabase,
+ query: args.query,
+ queryEmbedding: args.queryEmbedding,
+ ownerId: args.ownerId,
+ documentIds: args.documentIds,
+ matchCount: Math.max(args.matchCount, 48),
+ });
+ if (cards.length === 0) return { results: args.candidates, cards };
+
+ const memoryChunkResults = await loadChunksForMemoryCards(args.supabase, cards, args.ownerId);
+ const merged = mergeSearchResults(memoryChunkResults, args.candidates);
+ return {
+ results: applyMemoryCardBoosts(args.query, merged, cards),
+ cards,
+ };
+}
+
+async function attachDocumentRankingMetadata(
+ supabase: ReturnType,
+ results: SearchResult[],
+ ownerId?: string,
+) {
+ const documentIds = Array.from(new Set(results.map((result) => result.document_id)));
+ if (documentIds.length === 0) return results;
+
+ try {
+ const metadataRows = await fetchRelatedDocumentMetadata({
+ supabase,
+ ownerId,
+ documentIds,
+ });
+ const metadataByDocument = new Map(metadataRows.map((row) => [row.document_id, row]));
+ return results.map((result) => {
+ const metadata = metadataByDocument.get(result.document_id);
+ if (!metadata) return result;
+ return {
+ ...result,
+ document_labels: metadata.labels,
+ document_summary: metadata.summary,
+ };
+ });
+ } catch {
+ return results;
+ }
+}
+
+async function packAdjacentSourceContext(
+ supabase: ReturnType,
+ results: SearchResult[],
+ queryClass: RagQueryClass,
+ options: { crossDocument?: boolean } = {},
+) {
+ const contextLimit = options.crossDocument || queryClass === "comparison" || queryClass === "broad_summary" ? 8 : 5;
+ const targetResults = results.slice(0, contextLimit);
+ const documentIds = Array.from(new Set(targetResults.map((result) => result.document_id)));
+ const chunkIndexes = Array.from(
+ new Set(
+ targetResults.flatMap((result) => [result.chunk_index - 1, result.chunk_index + 1]).filter((index) => index >= 0),
+ ),
+ );
+ if (documentIds.length === 0 || chunkIndexes.length === 0) return results;
+
+ try {
+ const { data, error } = await supabase
+ .from("document_chunks")
+ .select("id,document_id,page_number,chunk_index,section_heading,content")
+ .in("document_id", documentIds)
+ .in("chunk_index", chunkIndexes)
+ .order("chunk_index", { ascending: true })
+ .limit(80);
+
+ if (error || !data?.length) return results;
+
+ const chunksByDocumentAndIndex = new Map();
+ for (const chunk of data) {
+ chunksByDocumentAndIndex.set(`${chunk.document_id}:${chunk.chunk_index}`, {
+ id: chunk.id,
+ section_heading: chunk.section_heading,
+ content: chunk.content,
+ });
+ }
+
+ const targetIds = new Set(targetResults.map((result) => result.id));
+ return results.map((result) => {
+ if (!targetIds.has(result.id)) return result;
+ const adjacent = [result.chunk_index - 1, result.chunk_index + 1]
+ .map((index) => chunksByDocumentAndIndex.get(`${result.document_id}:${index}`))
+ .filter((chunk): chunk is { id: string; section_heading: string | null; content: string } =>
+ Boolean(chunk && chunk.id !== result.id && chunk.content.trim()),
+ )
+ .map((chunk) => {
+ const heading = chunk.section_heading ? `${chunk.section_heading}: ` : "";
+ return compactContextText(`${heading}${chunk.content}`, 520);
+ });
+
+ if (adjacent.length === 0) return result;
+ return {
+ ...result,
+ adjacent_context: adjacent.join(" "),
+ };
+ });
+ } catch {
+ return results;
+ }
+}
+
+async function attachPageVisualEvidence(
+ supabase: ReturnType,
+ results: SearchResult[],
+): Promise {
+ const documentIds = Array.from(new Set(results.map((result) => result.document_id)));
+ const pageNumbers = Array.from(
+ new Set(results.map((result) => result.page_number).filter((page): page is number => Boolean(page))),
+ );
+ if (documentIds.length === 0 || pageNumbers.length === 0) return results;
+
+ const { data, error } = await supabase
+ .from("document_images")
+ .select(
+ "id,document_id,page_number,storage_path,caption,bbox,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata",
+ )
+ .in("document_id", documentIds)
+ .in("page_number", pageNumbers)
+ .eq("searchable", true)
+ .neq("image_type", "logo_decorative")
+ .order("clinical_relevance_score", { ascending: false })
+ .limit(80);
+
+ if (error || !data?.length) return results;
+
+ const imagesByPage = new Map();
+ for (const image of data) {
+ const metadata = safeRecord(image.metadata);
+ const rawTableText = metadataText(metadata, "table_text");
+ const tableText = metadataText(metadata, "table_text_snippet") ?? rawTableText;
+ const publicImage: ChunkImage = {
+ id: image.id,
+ page_number: image.page_number,
+ storage_path: image.storage_path,
+ caption: image.caption,
+ bbox: image.bbox,
+ image_type: image.image_type,
+ searchable: image.searchable,
+ clinical_relevance_score: image.clinical_relevance_score,
+ source_kind: image.source_kind,
+ sourceKind: image.source_kind,
+ tableLabel: metadataText(metadata, "table_label"),
+ tableTitle: metadataText(metadata, "table_title"),
+ tableRole: metadataText(metadata, "table_role"),
+ clinicalUseClass:
+ typeof metadata.clinical_use_class === "string"
+ ? (metadata.clinical_use_class as ClinicalImageUseClass)
+ : null,
+ clinicalUseReason:
+ typeof metadata.clinical_use_reason === "string" ? metadata.clinical_use_reason : null,
+ accessibleTableMarkdown:
+ typeof metadata.accessible_table_markdown === "string" ? metadata.accessible_table_markdown : rawTableText,
+ tableRows: Array.isArray(metadata.table_rows) ? (metadata.table_rows as string[][]) : null,
+ tableColumns: Array.isArray(metadata.table_columns) ? (metadata.table_columns as string[]) : null,
+ tableTextSnippet: tableText ? compactContextText(tableText, 500) : null,
+ labels: Array.isArray(image.labels) ? image.labels : [],
+ metadata,
+ };
+ const key = `${image.document_id}:${image.page_number}`;
+ imagesByPage.set(key, [...(imagesByPage.get(key) ?? []), publicImage]);
+ }
+
+ return results.map((result) => {
+ const pageImages = imagesByPage.get(`${result.document_id}:${result.page_number}`) ?? [];
+ if (pageImages.length === 0) return result;
+ const seen = new Set((result.images ?? []).map((image) => image.id));
+ const mergedImages = [...(result.images ?? []), ...pageImages.filter((image) => !seen.has(image.id))].slice(0, 4);
+ return { ...result, images: mergedImages };
+ });
+}
+
function shouldReturnTextFastPath(query: string, results: SearchResult[]) {
if (results.length === 0) return false;
+ const queryClass = classifyRagQuery(query).queryClass;
+ if (queryClass === "comparison") return false;
+ if (queryClass === "medication_dose_risk" && !results.slice(0, 5).some((result) => hasDoseEvidenceSupport(result))) {
+ return false;
+ }
+
const strongestScore = results.reduce((max, result) => Math.max(max, result.hybrid_score ?? result.similarity), 0);
const topTextRank = Math.max(...results.map((result) => result.text_rank ?? 0));
- return strongestScore >= 0.56 || topTextRank >= 0.05 || (hasDirectTitleSupport(query, results) && strongestScore >= 0.4);
+ return (
+ strongestScore >= 0.56 ||
+ topTextRank >= 0.05 ||
+ (hasDirectTitleSupport(query, results) && strongestScore >= 0.4) ||
+ (queryClass === "document_lookup" && hasDirectTitleSupport(query, results) && strongestScore >= 0.35)
+ );
+}
+
+function shouldAttemptDocumentLookupFastPath(queryClass: RagQueryClass) {
+ return queryClass === "document_lookup" || queryClass === "table_threshold";
}
-function boldClinicalSpecifics(input: string) {
- return input.replace(
- /\b(clozapine|lithium|ECT|FBC|ANC|myocarditis|neutropenia|metabolic|constipation|blood pressure|ECG|urgent|escalat\w*|withhold|cease|stop|\d+(?:\.\d+)?\s?(?:mg|mcg|g|mmol\/L|days?|weeks?|months?|hours?|minutes?|%))\b/gi,
- (match) => `**${match}**`,
+function shouldUseMemoryBeforeFastPath(queryClass: RagQueryClass) {
+ return queryClass === "table_threshold" || queryClass === "medication_dose_risk" || queryClass === "comparison";
+}
+
+function memoryCardAnswerLabel(card: DocumentMemoryCard) {
+ if (card.card_type === "table_row") return "Table evidence";
+ if (card.card_type === "threshold") return "Threshold/action";
+ if (card.card_type === "medication") return "Medication point";
+ if (card.card_type === "risk") return "Risk/escalation";
+ if (card.card_type === "workflow") return "Workflow step";
+ if (card.card_type === "section_summary") return "Section summary";
+ return "Source point";
+}
+
+function memoryCardAnswerScore(card: DocumentMemoryCard, query: string, queryClass: RagQueryClass) {
+ const content = sourceTextForDisplay(card.content);
+ if (!content) return -1;
+ const hasSpecificDoseEvidence =
+ /\b(?:mg|mcg|microgram|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|repeat(?:ing)? doses?|dose may be repeated|maximum \d|administer|titration|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test(
+ content,
+ );
+ if (
+ queryClass === "medication_dose_risk" &&
+ /\b(?:supporting information|relevant standards|references|document owner|authorisation|authorised by|published date|effective from|amendment|polypharmacy and high dose antipsychotic prescribing procedure)\b/i.test(
+ content,
+ ) &&
+ !hasSpecificDoseEvidence
+ ) {
+ return -1;
+ }
+ const normalizedContentTokens = new Set(splitBalancedWords(`${card.title} ${content}`));
+ const queryTokens = splitBalancedWords(query).filter((token) => token.length > 3);
+ const tokenHits = queryTokens.filter((token) => normalizedContentTokens.has(token)).length;
+ const typeBoost =
+ queryClass === "medication_dose_risk" && ["medication", "threshold", "table_row", "risk", "workflow"].includes(card.card_type)
+ ? 0.38
+ : queryClass === "table_threshold" && ["table_row", "threshold"].includes(card.card_type)
+ ? 0.32
+ : card.card_type === "section_summary"
+ ? 0.02
+ : 0.12;
+ const doseBoost =
+ queryClass === "medication_dose_risk" &&
+ /\b(?:dose|dosage|dosing|mg|mcg|microgram|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|route|titration|administer|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test(
+ content,
+ )
+ ? 0.42
+ : 0;
+ const lowValueTitlePenalty =
+ queryClass === "medication_dose_risk" &&
+ card.card_type === "section_summary" &&
+ !hasSpecificDoseEvidence
+ ? -0.35
+ : 0;
+
+ return tokenHits * 0.08 + typeBoost + doseBoost + (card.confidence ?? 0) * 0.08 + lowValueTitlePenalty;
+}
+
+function selectDiverseMemoryCards(cards: DocumentMemoryCard[], limit: number) {
+ const selected: Array<{ card: DocumentMemoryCard; tokens: Set }> = [];
+ for (const card of cards) {
+ const tokens = new Set(splitBalancedWords(card.content).filter((token) => token.length > 3 && !/^\d+$/.test(token)));
+ const duplicate = selected.some((item) => {
+ if (tokens.size === 0 || item.tokens.size === 0) return false;
+ const overlap = Array.from(tokens).filter((token) => item.tokens.has(token)).length;
+ return overlap / Math.min(tokens.size, item.tokens.size) >= 0.72;
+ });
+ if (duplicate) continue;
+ selected.push({ card, tokens });
+ if (selected.length >= limit) break;
+ }
+ return selected.map((item) => item.card);
+}
+
+function rankMemoryCardsForAnswer(cards: DocumentMemoryCard[], query: string, queryClass: RagQueryClass) {
+ return [...cards]
+ .map((card, index) => ({
+ card,
+ index,
+ score: memoryCardAnswerScore(card, query, queryClass),
+ }))
+ .filter((item) => item.score >= 0)
+ .sort((a, b) => b.score - a.score || a.index - b.index)
+ .map((item) => item.card);
+}
+
+type ExtractiveAnswerPoint = {
+ label: string;
+ text: string;
+ citationChunkIds: string[];
+ source: "dose" | "memory" | "quote";
+};
+
+function cleanExtractiveLine(line: string) {
+ return sourceTextForDisplay(line)
+ .replace(/^[-•]\s*/, "")
+ .replace(/^Agitation and Arousal:?\s+Pharmacological Management Guideline\s*/i, "")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+const extractiveLabelPattern =
+ /\b(?:Medication point|Table evidence|Threshold\/action|Risk\/escalation|Workflow step|Section summary|Source point|Dose detail|Monitoring)\s*:\s*/gi;
+
+function cleanExtractivePointText(value: string) {
+ return sourceTextForDisplay(value)
+ .replace(extractiveLabelPattern, " ")
+ .replace(/^[\s\-•:]+/, "")
+ .replace(/\s+[•]\s+/g, ". ")
+ .replace(/\s+-\s+(?=[A-Z][a-z])|(?:\s+-\s*)?(?:Medication point|Table evidence|Threshold\/action|Risk\/escalation|Workflow step|Section summary|Source point)\s*:\s*/gi, ". ")
+ .replace(/\s+/g, " ")
+ .replace(/\s+([,.;:])/g, "$1")
+ .replace(/(?:\.\s*){2,}/g, ". ")
+ .trim();
+}
+
+function sourcePointClauses(value: string, query: string) {
+ const tokens = splitBalancedWords(query).filter((token) => token.length > 3);
+ const clauses = cleanExtractivePointText(value)
+ .split(/(?<=[.!?])\s+|\s+[•]\s+|\s+\|\s+/)
+ .map((clause) => cleanExtractivePointText(clause))
+ .filter((clause) => clause.length >= 18 && !looksLikeJsonArtifact(clause));
+
+ return clauses
+ .map((clause, index) => {
+ const lower = clause.toLowerCase();
+ const tokenHits = tokens.filter((token) => lower.includes(token)).length;
+ const clinicalSignal = /\b(?:monitor|blood test|fbc|anc|level|baseline|review|urgent|escalat|dose|mg|withhold|cease|form|consent|commence|annual)\b/i.test(
+ clause,
+ )
+ ? 1
+ : 0;
+ const lengthPenalty = clause.length > 260 ? 0.8 : clause.length > 190 ? 0.25 : 0;
+ return { clause, score: tokenHits + clinicalSignal - lengthPenalty, index };
+ })
+ .sort((a, b) => b.score - a.score || a.index - b.index)
+ .map((item) => (item.clause.length <= 240 ? item.clause : `${item.clause.slice(0, 237).trim()}...`));
+}
+
+function bestNaturalSourcePoint(value: string, query: string) {
+ return sourcePointClauses(value, query)[0] ?? "";
+}
+
+function uniqueExtractivePoints(points: ExtractiveAnswerPoint[], limit: number) {
+ const seen = new Set();
+ const selected: ExtractiveAnswerPoint[] = [];
+ for (const point of points) {
+ const normalized = cleanExtractivePointText(point.text).toLowerCase();
+ const key = normalized.slice(0, 140);
+ if (!normalized || seen.has(key)) continue;
+ seen.add(key);
+ selected.push({ ...point, text: cleanExtractivePointText(point.text) });
+ if (selected.length >= limit) break;
+ }
+ return selected;
+}
+
+function extractMedicationDosePoints(results: SearchResult[], query: string, limit = 4): ExtractiveAnswerPoint[] {
+ const seen = new Set();
+ const points: ExtractiveAnswerPoint[] = [];
+ const coreQueryTokens = splitBalancedWords(query).filter(
+ (token) =>
+ token.length > 3 &&
+ !["dose", "dosing", "dosage", "medication", "medicine", "patient", "patients", "please"].includes(token),
);
+
+ for (const result of results) {
+ const resultText = `${result.title} ${result.content}`.toLowerCase();
+ if (coreQueryTokens.length && !coreQueryTokens.some((token) => resultText.includes(token))) continue;
+ const lines = sourceTextForDisplay(result.content)
+ .split(/\r?\n+/)
+ .map(cleanExtractiveLine)
+ .filter((line) => line.length >= 24);
+ const candidates = lines.flatMap((line) => {
+ if (/\b(?:supporting information|relevant standards|references|document owner|authorisation|published date|amendment)\b/i.test(line))
+ return [];
+ if (/\b(?:warnings? on the use|black box warning|food and drug administration|ann emerg med)\b/i.test(line)) return [];
+ const isMonitoring = /\b(?:monitor(?:ing)?|observations?|ecg|respiratory|every \d+|minutes?|hours?)\b/i.test(line);
+ const doseAnchor = line.search(
+ /\b(?:lorazepam|clonazepam|midazolam|risperidone|haloperidol|olanzapine|quetiapine|chlorpromazine|droperidol|promethazine|diazepam|\d+(?:\.\d+)?\s?mg|maximum\s+\d)/i,
+ );
+ const hasDoseAction = /\b(?:repeat(?:ing)? doses?|first dose|second dose|oral medication|im medication|maximum doses?|post im dose)\b/i.test(
+ line,
+ );
+ if (doseAnchor < 0 && !isMonitoring && !hasDoseAction) return [];
+ const headingMatch = [...line.matchAll(/\b(?:Recommended pharmacological treatment options|Repeating doses|Reviewing response|Monitoring|Oral Intramuscular)\s*:/gi)]
+ .filter((match) => (match.index ?? 0) < Math.max(doseAnchor, 0))
+ .at(-1);
+ const focused =
+ headingMatch?.index !== undefined
+ ? line.slice(headingMatch.index)
+ : doseAnchor > 80
+ ? line.slice(Math.max(0, doseAnchor - 60))
+ : line;
+ return [focused.replace(/^[^A-Za-z0-9]*(?:Appendix|Step)\s*\d*:?[^A-Za-z0-9]*/i, "").trim()];
+ });
+
+ for (const candidate of candidates) {
+ for (const clause of sourcePointClauses(candidate, query).slice(0, 3)) {
+ const text = clause.length <= 280 ? clause : `${clause.slice(0, 277).trim()}...`;
+ const key = text.toLowerCase();
+ if (seen.has(key)) continue;
+ seen.add(key);
+ const hasDrugDose = /\b(?:lorazepam|clonazepam|midazolam|risperidone|haloperidol|olanzapine|quetiapine|chlorpromazine|droperidol|promethazine|diazepam|\d+(?:\.\d+)?\s?mg|maximum\s+\d)\b/i.test(
+ text,
+ );
+ const label =
+ !hasDrugDose && /\b(?:monitor|observe|ecg|physiological|respiratory|minutes?|hours?)\b/i.test(text)
+ ? "Monitoring"
+ : "Dose detail";
+ points.push({ label, text, citationChunkIds: [result.id], source: "dose" });
+ if (points.length >= limit) return points;
+ }
+ }
+ }
+
+ return points;
+}
+
+function memoryCardToExtractivePoint(card: DocumentMemoryCard, query: string): ExtractiveAnswerPoint | null {
+ const text = bestNaturalSourcePoint(card.content, query);
+ if (!text) return null;
+ return {
+ label: memoryCardAnswerLabel(card),
+ text,
+ citationChunkIds: card.source_chunk_ids ?? [],
+ source: "memory",
+ };
+}
+
+function quoteToExtractivePoint(quote: QuoteCard, query: string): ExtractiveAnswerPoint | null {
+ const text = bestNaturalSourcePoint(`${quote.section_heading ? `${quote.section_heading}: ` : ""}${quote.quote}`, query);
+ if (!text) return null;
+ return {
+ label: quote.section_heading ?? "Source quote",
+ text,
+ citationChunkIds: [quote.chunk_id],
+ source: "quote",
+ };
+}
+
+function naturalAnswerLead(query: string, queryClass: RagQueryClass, points: ExtractiveAnswerPoint[]) {
+ if (/\bclozapine\b/i.test(query) && /\bmonitor/i.test(query)) {
+ return "The retrieved clozapine sources support a monitoring-focused answer, but the selected excerpts are strongest for specific source points rather than a full monitoring schedule.";
+ }
+ if (queryClass === "medication_dose_risk") {
+ return "The retrieved medication/risk sources support these practical points.";
+ }
+ if (queryClass === "table_threshold") {
+ return "The retrieved table or threshold evidence supports these points.";
+ }
+ if (points.length === 1) return "The strongest retrieved source supports this point.";
+ return "The strongest retrieved sources support these points.";
+}
+
+function formatNaturalPoint(point: ExtractiveAnswerPoint) {
+ const text = cleanExtractivePointText(point.text).replace(/[.;,\s]+$/, "");
+ if (!text) return "";
+ if (/^(the|this|these|there|monitor|review|ensure|commence|copy|prescribe|annual|time since|blood test)\b/i.test(text)) {
+ return `${text}.`;
+ }
+ if (point.label === "Monitoring") return `Monitoring evidence: ${text}.`;
+ if (point.label === "Dose detail") return `Dose evidence: ${text}.`;
+ return `${text}.`;
+}
+
+function buildNaturalExtractiveAnswer(args: {
+ query: string;
+ queryClass: RagQueryClass;
+ points: ExtractiveAnswerPoint[];
+ sourceCount: number;
+}) {
+ const points = uniqueExtractivePoints(args.points, 3);
+ if (!points.length) {
+ return {
+ answer:
+ "The indexed source passages matched the question, but no concise source sentence could be extracted. Open the cited sources before relying on this result.",
+ body:
+ "No concise source sentence could be extracted from the selected passages. Use the linked citations to inspect the source text.",
+ citationChunkIds: [] as string[],
+ };
+ }
+
+ const lead = naturalAnswerLead(args.query, args.queryClass, points);
+ const pointSentences = points.map(formatNaturalPoint).filter(Boolean);
+ const caveat =
+ args.queryClass === "medication_dose_risk" && /\bmonitor/i.test(args.query)
+ ? "If you need the complete monitoring schedule, open the linked source pages and check the surrounding table or section."
+ : "";
+ const answer = [lead, ...pointSentences, caveat].filter(Boolean).join(" ");
+ const body = [lead, ...pointSentences].filter(Boolean).join(" ");
+
+ return {
+ answer: boldHighYieldClinicalText(answer, args.query),
+ body: boldHighYieldClinicalText(body, args.query),
+ citationChunkIds: Array.from(new Set(points.flatMap((point) => point.citationChunkIds))),
+ };
}
function buildExtractiveAnswer(args: {
query: string;
+ queryClass: RagQueryClass;
results: SearchResult[];
quoteCards: QuoteCard[];
documentBreakdown: RagAnswer["documentBreakdown"];
@@ -498,25 +1433,48 @@ function buildExtractiveAnswer(args: {
routeReason: string;
timings: RagAnswer["latencyTimings"];
}) {
- const quoteCards = args.quoteCards.length ? args.quoteCards.slice(0, 5) : extractQuoteCards(args.results, args.query, 5);
+ const quoteCards = args.quoteCards.length
+ ? args.quoteCards.slice(0, 5)
+ : extractQuoteCards(args.results, args.query, 5);
+ const memoryCards = rankMemoryCardsForAnswer(collectMemoryCards(args.results, 16), args.query, args.queryClass).slice(0, 10);
const citations = compactCitations(args.results).slice(0, Math.max(quoteCards.length, 1));
const citationIds = new Set(citations.map((citation) => citation.chunk_id));
+ const resultById = new Map(args.results.map((result) => [result.id, result]));
+ for (const card of memoryCards) {
+ for (const chunkId of card.source_chunk_ids ?? []) {
+ if (citationIds.has(chunkId)) continue;
+ const source = resultById.get(chunkId);
+ if (!source) continue;
+ citations.push(resultCitation(source));
+ citationIds.add(chunkId);
+ }
+ }
for (const quote of quoteCards) {
- if (!citationIds.has(quote.chunk_id)) citations.push(resultCitation(args.results.find((result) => result.id === quote.chunk_id)!));
+ if (!citationIds.has(quote.chunk_id))
+ citations.push(resultCitation(args.results.find((result) => result.id === quote.chunk_id)!));
citationIds.add(quote.chunk_id);
}
- const bullets = quoteCards.slice(0, 5).map((quote) => {
- const section = quote.section_heading ? `${quote.section_heading}: ` : "";
- return `- ${boldClinicalSpecifics(`${section}${quote.quote}`)}`;
+ const dosePoints =
+ args.queryClass === "medication_dose_risk" ? extractMedicationDosePoints(args.results, args.query, 4) : [];
+ const remainingPointSlots = Math.max(0, 5 - dosePoints.length);
+ const memoryPointLimit = Math.min(remainingPointSlots, memoryCards.length >= 4 ? 3 : 2);
+ const memoryPoints = selectDiverseMemoryCards(memoryCards, memoryPointLimit)
+ .map((card) => memoryCardToExtractivePoint(card, args.query))
+ .filter((point): point is ExtractiveAnswerPoint => Boolean(point));
+ const quotePoints = quoteCards
+ .slice(0, Math.max(0, 5 - dosePoints.length - memoryPoints.length))
+ .map((quote) => quoteToExtractivePoint(quote, args.query))
+ .filter((point): point is ExtractiveAnswerPoint => Boolean(point));
+ const naturalAnswer = buildNaturalExtractiveAnswer({
+ query: args.query,
+ queryClass: args.queryClass,
+ points: [...dosePoints, ...memoryPoints, ...quotePoints],
+ sourceCount: args.results.length,
});
- const answer = bullets.length
- ? bullets.join("\n")
- : "The indexed source passages matched the question, but no concise source sentence could be extracted. Open the cited sources before relying on this result.";
-
return {
- answer,
+ answer: naturalAnswer.answer,
grounded: citations.length > 0,
confidence: deriveConfidence(args.results, citations.length),
citations: citations.slice(0, 5),
@@ -524,13 +1482,14 @@ function buildExtractiveAnswer(args: {
modelUsed: null,
routingMode: "extractive",
routingReason: args.routeReason,
+ queryClass: args.queryClass,
latencyTimings: args.timings,
- answerSections: bullets.length
+ answerSections: naturalAnswer.citationChunkIds.length
? [
{
- heading: "High-yield source points",
- body: bullets.join("\n"),
- citation_chunk_ids: quoteCards.map((quote) => quote.chunk_id),
+ heading: "Direct source-backed answer",
+ body: naturalAnswer.body,
+ citation_chunk_ids: naturalAnswer.citationChunkIds,
},
]
: [],
@@ -543,28 +1502,53 @@ function buildExtractiveAnswer(args: {
conflictsOrGaps: args.conflictsOrGaps,
smartPanel: args.smartPanel,
relatedDocuments: args.relatedDocuments,
+ memoryCardsUsed: memoryCards,
+ indexingVersion: ragDeepMemoryVersion,
+ indexingQuality: buildIndexingQuality(args.results, memoryCards),
+ scoreExplanations: buildAnswerScoreExplanations(args.results),
} satisfies RagAnswer;
}
+function isUnusableGeneratedAnswer(answer: Pick) {
+ const normalized = normalizeSectionText(answer.answer ?? "");
+ if (!normalized) return true;
+ if (normalized === machineReadableFallbackAnswer) return true;
+ if (answer.routingReason === "structured_parse_fallback") return true;
+ return looksLikeJsonArtifact(normalized);
+}
+
export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
const cached = getCachedSearch(args);
if (cached) return cached;
const supabase = createAdminClient();
- const documentFilterList = args.documentIds?.length ? args.documentIds : args.documentId ? [args.documentId] : undefined;
+ const queryClassification = classifyRagQuery(args.query);
+ const documentFilterList = args.documentIds?.length
+ ? args.documentIds
+ : args.documentId
+ ? [args.documentId]
+ : undefined;
const telemetry: SearchTelemetry = {
search_cache_hit: false,
+ query_class: queryClassification.queryClass,
text_fast_path_latency_ms: 0,
embedding_skipped: false,
embedding_latency_ms: 0,
embedding_cache_hit: false,
supabase_rpc_latency_ms: 0,
rerank_latency_ms: 0,
+ memory_card_count: 0,
+ memory_top_score: 0,
+ weighted_top_score: 0,
+ rrf_top_score: 0,
};
const expandedQuery = expandClinicalQuery(args.query);
const textSearchQuery = buildClinicalTextSearchQuery(args.query);
- const candidateCount = Math.max((args.topK ?? 8) * 5, 48);
+ const candidateMultiplier = queryClassification.queryClass === "comparison" ? 7 : 5;
+ const candidateFloor = queryClassification.queryClass === "comparison" ? 72 : 48;
+ const candidateCount = Math.max((args.topK ?? 8) * candidateMultiplier, candidateFloor);
+ const maxResultsPerDocument = queryClassification.queryClass === "comparison" ? 2 : 4;
const minSimilarity = args.minSimilarity ?? 0.15;
let textFastResults: SearchResult[] = [];
@@ -580,17 +1564,100 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
if (!textError && textData?.length) {
const rerankStartedAt = Date.now();
- textFastResults = diversifySearchResults(rankClinicalResults(args.query, textData as SearchResult[]), args.topK ?? 8);
+ const textCandidates = await attachDocumentRankingMetadata(supabase, textData as SearchResult[], args.ownerId);
+ const baseTextResults = diversifySearchResults(
+ rankClinicalResults(args.query, textCandidates),
+ args.topK ?? 8,
+ maxResultsPerDocument,
+ true,
+ );
+
+ if (shouldReturnTextFastPath(args.query, baseTextResults) && !shouldUseMemoryBeforeFastPath(queryClassification.queryClass)) {
+ textFastResults = await attachPageVisualEvidence(supabase, baseTextResults);
+ telemetry.rerank_latency_ms += Date.now() - rerankStartedAt;
+ telemetry.embedding_skipped = true;
+ telemetry.retrieval_strategy = "text_fast_path";
+ recordSearchScoreTelemetry(telemetry, textFastResults);
+ setCachedSearch(args, textFastResults, telemetry);
+ return { results: textFastResults, telemetry };
+ }
+
+ const memoryBoost = await withMemoryBoostedCandidates({
+ supabase,
+ query: args.query,
+ candidates: textCandidates,
+ ownerId: args.ownerId,
+ documentIds: documentFilterList,
+ matchCount: candidateCount,
+ });
+ telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length);
+ telemetry.memory_top_score = Math.max(telemetry.memory_top_score ?? 0, ...memoryBoost.cards.map(memoryCardChunkScore));
+ textFastResults = diversifySearchResults(
+ rankClinicalResults(args.query, memoryBoost.results),
+ args.topK ?? 8,
+ maxResultsPerDocument,
+ true,
+ );
+ textFastResults = await attachPageVisualEvidence(supabase, textFastResults);
telemetry.rerank_latency_ms += Date.now() - rerankStartedAt;
if (shouldReturnTextFastPath(args.query, textFastResults)) {
telemetry.embedding_skipped = true;
telemetry.retrieval_strategy = "text_fast_path";
+ recordSearchScoreTelemetry(telemetry, textFastResults);
setCachedSearch(args, textFastResults, telemetry);
return { results: textFastResults, telemetry };
}
}
+ if (shouldAttemptDocumentLookupFastPath(queryClassification.queryClass)) {
+ const documentLookupStartedAt = Date.now();
+ const documentLookupData = await searchDocumentLookupFastPath({
+ supabase,
+ query: args.query,
+ ownerId: args.ownerId,
+ documentIds: documentFilterList,
+ matchCount: candidateCount,
+ });
+ telemetry.supabase_rpc_latency_ms += Date.now() - documentLookupStartedAt;
+
+ if (documentLookupData.length > 0) {
+ const rerankStartedAt = Date.now();
+ const documentLookupCandidates = await attachDocumentRankingMetadata(
+ supabase,
+ mergeSearchResults(documentLookupData, textFastResults),
+ args.ownerId,
+ );
+ const memoryBoost = await withMemoryBoostedCandidates({
+ supabase,
+ query: args.query,
+ candidates: documentLookupCandidates,
+ ownerId: args.ownerId,
+ documentIds: documentFilterList,
+ matchCount: candidateCount,
+ });
+ telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length);
+ telemetry.memory_top_score = Math.max(
+ telemetry.memory_top_score ?? 0,
+ ...memoryBoost.cards.map(memoryCardChunkScore),
+ );
+ const documentLookupResults = await attachPageVisualEvidence(
+ supabase,
+ diversifySearchResults(rankClinicalResults(args.query, memoryBoost.results), args.topK ?? 8, maxResultsPerDocument, true),
+ );
+ telemetry.rerank_latency_ms += Date.now() - rerankStartedAt;
+
+ if (shouldReturnTextFastPath(args.query, documentLookupResults)) {
+ telemetry.embedding_skipped = true;
+ telemetry.retrieval_strategy = "document_lookup_fast_path";
+ recordSearchScoreTelemetry(telemetry, documentLookupResults);
+ setCachedSearch(args, documentLookupResults, telemetry);
+ return { results: documentLookupResults, telemetry };
+ }
+ textFastResults = mergeSearchResults(documentLookupResults, textFastResults);
+ }
+ }
+
const embeddingStartedAt = Date.now();
const { embedding, cacheHit } = await embedTextWithTelemetry(expandedQuery);
telemetry.embedding_latency_ms = Date.now() - embeddingStartedAt;
@@ -610,9 +1677,25 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
if (!hybridError) {
const rerankStartedAt = Date.now();
const merged = mergeSearchResults((hybridData ?? []) as SearchResult[], textFastResults);
- const results = diversifySearchResults(rankClinicalResults(args.query, merged), args.topK ?? 8);
+ const mergedWithMetadata = await attachDocumentRankingMetadata(supabase, merged, args.ownerId);
+ const memoryBoost = await withMemoryBoostedCandidates({
+ supabase,
+ query: args.query,
+ candidates: mergedWithMetadata,
+ queryEmbedding: embedding,
+ ownerId: args.ownerId,
+ documentIds: documentFilterList,
+ matchCount: candidateCount,
+ });
+ telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length);
+ telemetry.memory_top_score = Math.max(telemetry.memory_top_score ?? 0, ...memoryBoost.cards.map(memoryCardChunkScore));
+ const results = await attachPageVisualEvidence(
+ supabase,
+ diversifySearchResults(rankClinicalResults(args.query, memoryBoost.results), args.topK ?? 8, maxResultsPerDocument, true),
+ );
telemetry.rerank_latency_ms += Date.now() - rerankStartedAt;
telemetry.retrieval_strategy = "hybrid";
+ recordSearchScoreTelemetry(telemetry, results);
setCachedSearch(args, results, telemetry);
return { results, telemetry };
}
@@ -637,9 +1720,29 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
telemetry.supabase_rpc_latency_ms += Date.now() - fallbackRpcStartedAt;
const rerankStartedAt = Date.now();
- const results = diversifySearchResults(rankClinicalResults(args.query, mergeSearchResults(resultSets.flat(), textFastResults)), args.topK ?? 8);
+ const mergedWithMetadata = await attachDocumentRankingMetadata(
+ supabase,
+ mergeSearchResults(resultSets.flat(), textFastResults),
+ args.ownerId,
+ );
+ const memoryBoost = await withMemoryBoostedCandidates({
+ supabase,
+ query: args.query,
+ candidates: mergedWithMetadata,
+ queryEmbedding: embedding,
+ ownerId: args.ownerId,
+ documentIds: documentFilterList,
+ matchCount: candidateCount,
+ });
+ telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length);
+ telemetry.memory_top_score = Math.max(telemetry.memory_top_score ?? 0, ...memoryBoost.cards.map(memoryCardChunkScore));
+ const results = await attachPageVisualEvidence(
+ supabase,
+ diversifySearchResults(rankClinicalResults(args.query, memoryBoost.results), args.topK ?? 8, maxResultsPerDocument, true),
+ );
telemetry.rerank_latency_ms += Date.now() - rerankStartedAt;
telemetry.retrieval_strategy = "vector_fallback";
+ recordSearchScoreTelemetry(telemetry, results);
setCachedSearch(args, results, telemetry);
return { results, telemetry };
}
@@ -662,15 +1765,39 @@ export async function searchChunks(args: SearchChunksArgs) {
return results;
}
+function compactContextText(text: string, limit: number) {
+ const compact = sourceTextForModel(text).replace(/\s+/g, " ").trim();
+ if (compact.length <= limit) return compact;
+ return `${compact.slice(0, limit - 3).trim()}...`;
+}
+
export function buildRagSourceBlock(results: SearchResult[]) {
return results
.map((result, index) => {
const page = result.page_number ? `page ${result.page_number}` : "page unavailable";
- const searchableImages = result.images?.filter(
- (image) => image.searchable !== false && image.image_type !== "logo_decorative",
- );
+ const searchableImages = result.images?.filter((image) => isClinicalImageEvidence(image));
const images = searchableImages?.length
- ? `\nImages: ${searchableImages.map((image) => image.caption).join(" | ")}`
+ ? `\nImages: ${searchableImages
+ .map((image) =>
+ [
+ image.tableLabel,
+ image.tableTitle,
+ image.caption,
+ image.tableTextSnippet ? `Table text: ${compactContextText(image.tableTextSnippet, 320)}` : "",
+ ]
+ .filter(Boolean)
+ .join(" - "),
+ )
+ .join(" | ")}`
+ : "";
+ const adjacentContext = result.adjacent_context
+ ? `\nNearby context from the same source: ${compactContextText(result.adjacent_context, 900)}`
+ : "";
+ const memoryCards = result.memory_cards?.length
+ ? `\nStructured memory: ${result.memory_cards
+ .slice(0, 3)
+ .map((card) => `${card.card_type}: ${compactContextText(card.content, 300)}`)
+ .join(" | ")}`
: "";
return [
[
@@ -678,14 +1805,16 @@ export function buildRagSourceBlock(results: SearchResult[]) {
`citation_chunk_id: ${result.id}`,
`document_id: ${result.document_id}`,
].join("\n"),
- result.content,
+ compactContextText(result.content, 1800),
+ adjacentContext,
+ memoryCards,
images,
].join("\n");
})
.join("\n\n---\n\n");
}
-export function parseAnswerJson(raw: string, results: SearchResult[]): RagAnswer {
+export function parseAnswerJson(raw: string, results: SearchResult[], query?: string): RagAnswer {
try {
const parsed = answerJsonSchema.parse(JSON.parse(raw));
const citations = sanitizeCitations(parsed.citations, results);
@@ -693,13 +1822,20 @@ export function parseAnswerJson(raw: string, results: SearchResult[]): RagAnswer
? deriveConfidence(results, citations.length)
: clampConfidence("low", deriveConfidence(results, citations.length));
const confidence = clampConfidence(parsed.confidence, derivedConfidence);
+ const parsedAnswer = parsed.answer ?? "";
+ const nonArtifactParsedAnswer = parsedAnswer.trim() && !looksLikeJsonArtifact(parsedAnswer) ? parsedAnswer : "";
+ const sanitizedAnswer =
+ sanitizeStructuredText(parsedAnswer, { minLength: 8, minTokens: 2 }) ||
+ nonArtifactParsedAnswer ||
+ machineReadableFallbackAnswer;
+ const answerSections = sanitizeAnswerSections(parsed.answerSections, results, query);
return {
- answer: parsed.answer ?? raw,
+ answer: boldHighYieldClinicalText(sanitizedAnswer, query),
grounded: citations.length > 0 && confidence !== "unsupported",
confidence,
citations,
sources: results,
- answerSections: sanitizeAnswerSections(parsed.answerSections, results),
+ answerSections,
conflictsOrGaps: sanitizeConflictsOrGaps(parsed.conflictsOrGaps, results),
quoteCards: sanitizeQuoteCards(parsed.quoteCards, results),
visualEvidence: [],
@@ -707,7 +1843,7 @@ export function parseAnswerJson(raw: string, results: SearchResult[]): RagAnswer
documentBreakdown: [],
};
} catch {
- return safeFallbackAnswer(raw, results);
+ return safeFallbackAnswer(raw, results, query);
}
}
@@ -727,14 +1863,26 @@ export async function answerQuestionWithScope(args: {
const startedAt = Date.now();
const cachedAnswer = getCachedAnswer(args, startedAt);
if (cachedAnswer) {
+ const cachedSources = annotateSearchResults(args.query, cachedAnswer.sources ?? []);
+ const cachedRelevance = cachedAnswer.relevance ?? buildEvidenceRelevance(args.query, cachedSources);
await args.onProgress?.({
stage: "cached",
message: "Using a recent cited answer for this exact query and document scope.",
mode: cachedAnswer.routingMode,
model: cachedAnswer.modelUsed,
reason: cachedAnswer.routingReason,
+ resultCount: cachedSources.length,
+ visibleSourceCount: cachedSources.length,
+ directSourceCount: cachedRelevance.directSourceCount,
+ weakSourceCount: cachedRelevance.weakSourceCount,
+ relevance: cachedRelevance,
});
- return cachedAnswer;
+ return {
+ ...cachedAnswer,
+ sources: cachedSources,
+ relevance: cachedRelevance,
+ smartPanel: cachedAnswer.smartPanel ? { ...cachedAnswer.smartPanel, relevance: cachedRelevance } : cachedAnswer.smartPanel,
+ };
}
const searchStartedAt = Date.now();
@@ -747,16 +1895,51 @@ export async function answerQuestionWithScope(args: {
minSimilarity: 0.12,
skipCache: args.skipCache,
});
- const results = normalizeSearchResults(search.results);
+ const queryClass = search.telemetry.query_class ?? classifyRagQuery(args.query).queryClass;
+ const answerRanking = rankAnswerEvidence(args.query, normalizeSearchResults(search.results), queryClass);
+ const results = annotateSearchResults(args.query, answerRanking.rankedResults);
+ const crossDocumentPlan = buildCrossDocumentSynthesisPlan(args.query, results, queryClass);
+ const answerInputResults = crossDocumentPlan.enabled ? crossDocumentPlan.results : results;
+ const relevance = buildEvidenceRelevance(args.query, answerInputResults);
+ const crossDocumentFusionBrief = crossDocumentPlan.enabled
+ ? buildCrossDocumentFusionBrief(args.query, answerInputResults)
+ : null;
+ const answerRankMetadata = {
+ answer_rank_top_score: answerRanking.topScore,
+ answer_ranked_source_count: answerRanking.rankedSourceCount,
+ answer_rank_strategy: answerRanking.strategy,
+ answer_rank_query_class: answerRanking.queryClass,
+ cross_document_synthesis: crossDocumentPlan.enabled,
+ cross_document_reason: crossDocumentPlan.reason,
+ cross_document_count: crossDocumentPlan.documentCount,
+ cross_document_selected_count: crossDocumentPlan.selectedDocumentCount,
+ cross_document_selected_source_count: crossDocumentPlan.selectedSourceCount,
+ cross_document_fusion_bullets: crossDocumentFusionBrief?.bulletCount ?? 0,
+ cross_document_fusion_source_chunk_ids: crossDocumentFusionBrief?.sourceChunkIds ?? [],
+ };
const searchLatencyMs = Date.now() - searchStartedAt;
- const quoteCards = extractQuoteCards(results, args.query);
- const documentBreakdown = buildDocumentBreakdown(results, quoteCards);
- const smartPanel = buildSmartPanel(args.query, results);
- const evidenceSummary = buildEvidenceSummary(results, quoteCards);
- const sourceCoverage = buildSourceCoverage(results);
- const conflictsOrGaps = detectConflictsOrGaps(results);
- const visualEvidence = buildVisualEvidence(results);
- const bestSource = selectBestSourceRecommendation(results, quoteCards);
+ const quoteCards = extractQuoteCards(answerInputResults, args.query);
+ const documentBreakdown = buildDocumentBreakdown(answerInputResults, quoteCards);
+ const smartPanel = buildSmartPanel(args.query, answerInputResults);
+ const evidenceSummary = buildEvidenceSummary(answerInputResults, quoteCards);
+ const sourceCoverage = buildSourceCoverage(answerInputResults);
+ const conflictsOrGaps = detectConflictsOrGaps(answerInputResults);
+ const visualEvidence = buildVisualEvidence(answerInputResults);
+ const bestSource = selectBestSourceRecommendation(answerInputResults, quoteCards);
+ const memoryCardsUsed = collectMemoryCards(answerInputResults);
+ const indexingQuality = buildIndexingQuality(answerInputResults, memoryCardsUsed);
+ const memoryLogMetadata = {
+ memory_card_count: memoryCardsUsed.length,
+ memory_top_score: Number(
+ Math.max(0, ...results.map((result) => result.memory_score ?? 0), ...memoryCardsUsed.map(memoryCardChunkScore)).toFixed(4),
+ ),
+ indexing_version: ragDeepMemoryVersion,
+ indexing_extraction_quality: indexingQuality.extractionQuality,
+ indexing_stale: indexingQuality.stale,
+ };
+ const answerScoreExplanations = buildAnswerScoreExplanations(answerInputResults);
+ const scoreLogMetadata = scoreExplanationLogMetadata(answerScoreExplanations);
+ const emptyPanel = buildSmartPanel(args.query, []);
const relatedDocumentsPromise = buildRelatedDocumentsSafe({
query: args.query,
results,
@@ -764,15 +1947,41 @@ export async function answerQuestionWithScope(args: {
});
const route = chooseAnswerRoute({
query: args.query,
- results,
+ results: answerInputResults,
+ queryClass,
conflictsOrGaps,
fastModel: env.OPENAI_FAST_ANSWER_MODEL,
strongModel: env.OPENAI_STRONG_ANSWER_MODEL,
});
+ const buildCurrentSmartApiPlan = (
+ mode: RagAnswer["routingMode"] = route.mode,
+ reason = route.reason,
+ planResults = answerInputResults,
+ ) =>
+ buildSmartRagApiPlan({
+ query: args.query,
+ queryClass,
+ results: planResults,
+ routeMode: mode,
+ routeReason: reason,
+ retrievalStrategy: search.telemetry.retrieval_strategy,
+ });
+ const smartApiPlan = buildCurrentSmartApiPlan();
+ const smartApiLogMetadata = (plan: SmartRagApiPlan) => ({
+ smart_api_intent: plan.intent,
+ smart_api_response_mode: plan.responseMode,
+ smart_api_latency_plan: plan.latencyPlan,
+ smart_api_source_link_count: plan.sourceLinkCount,
+ });
await args.onProgress?.({
stage: "retrieved",
- message: `Retrieved ${results.length} candidate source${results.length === 1 ? "" : "s"}.`,
+ message: `${relevance.label}: retrieved ${results.length} candidate source${results.length === 1 ? "" : "s"}.`,
resultCount: results.length,
+ visibleSourceCount: answerInputResults.length,
+ directSourceCount: relevance.directSourceCount,
+ weakSourceCount: relevance.weakSourceCount,
+ timingMs: searchLatencyMs,
+ relevance,
});
await args.onProgress?.({
stage: "routing",
@@ -783,16 +1992,16 @@ export async function answerQuestionWithScope(args: {
mode: route.mode,
model: route.model,
reason: route.reason,
+ smartApiPlan,
});
if (route.mode === "unsupported") {
const relatedDocuments = await relatedDocumentsPromise;
- const emptyPanel = buildSmartPanel(args.query, []);
const unsupportedWithNearbySources = results.length > 0;
const answer = {
answer: unsupportedWithNearbySources
- ? "I found nearby indexed passages, but they are not strong enough to support a reliable answer. Try rephrasing the question or selecting a more relevant document."
- : "I could not find enough support in the indexed documents to answer this question. Upload or index a relevant guideline, then search again.",
+ ? "I found nearby indexed passages, but they are not strong enough to support a reliable answer. Try refining the query or selecting a more relevant document."
+ : "I could not find enough support in the indexed documents to answer this query. Upload or index a relevant guideline, then search again.",
grounded: false,
confidence: "unsupported",
citations: [],
@@ -800,6 +2009,7 @@ export async function answerQuestionWithScope(args: {
modelUsed: null,
routingMode: route.mode,
routingReason: route.reason,
+ queryClass,
latencyTimings: {
search_cache_hit: search.telemetry.search_cache_hit,
text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
@@ -808,6 +2018,7 @@ export async function answerQuestionWithScope(args: {
embedding_cache_hit: search.telemetry.embedding_cache_hit,
supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ context_pack_latency_ms: 0,
search_latency_ms: searchLatencyMs,
generation_latency_ms: 0,
total_latency_ms: Date.now() - startedAt,
@@ -821,44 +2032,60 @@ export async function answerQuestionWithScope(args: {
sourceCoverage: unsupportedWithNearbySources ? sourceCoverage : emptyPanel.sourceCoverage,
conflictsOrGaps,
smartPanel: unsupportedWithNearbySources
- ? { ...smartPanel, bestSource, relatedDocuments }
- : { ...emptyPanel, relatedDocuments },
+ ? { ...smartPanel, relevance, bestSource, relatedDocuments }
+ : { ...emptyPanel, relevance, relatedDocuments },
relatedDocuments,
+ relevance,
+ memoryCardsUsed: unsupportedWithNearbySources ? memoryCardsUsed : [],
+ indexingVersion: ragDeepMemoryVersion,
+ indexingQuality,
+ smartApiPlan,
+ scoreExplanations: answerScoreExplanations,
} satisfies RagAnswer;
- if (args.logQuery !== false) await logRagQuery({
- owner_id: args.ownerId ?? null,
- query: args.query,
- answer: answer.answer,
- source_chunk_ids: results.map((result) => result.id),
- model: null,
- metadata: {
- document_id: args.documentId ?? null,
- document_ids: args.documentIds ?? null,
- grounded: answer.grounded,
- confidence: answer.confidence,
- routing_mode: route.mode,
- routing_reason: route.reason,
- model_used: null,
- retrieved_candidate_count: results.length,
- cited_chunk_count: 0,
- quote_count: answer.quoteCards?.length ?? 0,
- visual_evidence_count: answer.visualEvidence?.length ?? 0,
- search_cache_hit: search.telemetry.search_cache_hit,
- text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
- embedding_skipped: search.telemetry.embedding_skipped,
- embedding_latency_ms: search.telemetry.embedding_latency_ms,
- embedding_cache_hit: search.telemetry.embedding_cache_hit,
- supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
- rerank_latency_ms: search.telemetry.rerank_latency_ms,
- retrieval_strategy: search.telemetry.retrieval_strategy,
- search_latency_ms: searchLatencyMs,
- generation_latency_ms: 0,
- total_latency_ms: answer.latencyTimings.total_latency_ms,
- evidence_summary: answer.evidenceSummary,
- related_document_count: relatedDocuments.length,
- },
- });
+ if (args.logQuery !== false)
+ await logRagQuery({
+ owner_id: args.ownerId ?? null,
+ query: args.query,
+ answer: answer.answer,
+ source_chunk_ids: answerInputResults.map((result) => result.id),
+ model: null,
+ metadata: {
+ document_id: args.documentId ?? null,
+ document_ids: args.documentIds ?? null,
+ grounded: answer.grounded,
+ confidence: answer.confidence,
+ routing_mode: route.mode,
+ routing_reason: route.reason,
+ query_class: queryClass,
+ fallback_reason: fallbackReasonFromRouting(route.reason),
+ model_used: null,
+ retrieved_candidate_count: results.length,
+ ...smartApiLogMetadata(smartApiPlan),
+ ...answerRankMetadata,
+ ...memoryLogMetadata,
+ ...scoreLogMetadata,
+ cited_chunk_count: 0,
+ quote_count: answer.quoteCards?.length ?? 0,
+ visual_evidence_count: answer.visualEvidence?.length ?? 0,
+ search_cache_hit: search.telemetry.search_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ retrieval_strategy: search.telemetry.retrieval_strategy,
+ weighted_top_score: search.telemetry.weighted_top_score,
+ rrf_top_score: search.telemetry.rrf_top_score,
+ search_latency_ms: searchLatencyMs,
+ generation_latency_ms: 0,
+ total_latency_ms: answer.latencyTimings.total_latency_ms,
+ evidence_summary: answer.evidenceSummary,
+ source_coverage: answer.sourceCoverage,
+ related_document_count: relatedDocuments.length,
+ },
+ });
setCachedAnswer(args, answer);
return answer;
@@ -866,9 +2093,10 @@ export async function answerQuestionWithScope(args: {
if (route.mode === "extractive") {
const relatedDocuments = await relatedDocumentsPromise;
- const answer = buildExtractiveAnswer({
+ const answer: RagAnswer = buildExtractiveAnswer({
query: args.query,
- results,
+ queryClass,
+ results: answerInputResults,
quoteCards,
documentBreakdown,
evidenceSummary,
@@ -876,7 +2104,7 @@ export async function answerQuestionWithScope(args: {
conflictsOrGaps,
visualEvidence,
bestSource,
- smartPanel: { ...smartPanel, bestSource, relatedDocuments },
+ smartPanel: { ...smartPanel, relevance, bestSource, relatedDocuments },
relatedDocuments,
routeReason: route.reason,
timings: {
@@ -887,45 +2115,60 @@ export async function answerQuestionWithScope(args: {
embedding_cache_hit: search.telemetry.embedding_cache_hit,
supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ context_pack_latency_ms: 0,
search_latency_ms: searchLatencyMs,
generation_latency_ms: 0,
total_latency_ms: Date.now() - startedAt,
},
});
-
- if (args.logQuery !== false) await logRagQuery({
- owner_id: args.ownerId ?? null,
- query: args.query,
- answer: answer.answer,
- source_chunk_ids: results.map((result) => result.id),
- model: null,
- metadata: {
- document_id: args.documentId ?? null,
- document_ids: args.documentIds ?? null,
- grounded: answer.grounded,
- confidence: answer.confidence,
- routing_mode: answer.routingMode,
- routing_reason: answer.routingReason,
- model_used: null,
- retrieved_candidate_count: results.length,
- cited_chunk_count: answer.citations.length,
- quote_count: answer.quoteCards?.length ?? 0,
- visual_evidence_count: answer.visualEvidence?.length ?? 0,
- related_document_count: relatedDocuments.length,
- search_cache_hit: search.telemetry.search_cache_hit,
- text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
- embedding_skipped: search.telemetry.embedding_skipped,
- embedding_latency_ms: search.telemetry.embedding_latency_ms,
- embedding_cache_hit: search.telemetry.embedding_cache_hit,
- supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
- rerank_latency_ms: search.telemetry.rerank_latency_ms,
- retrieval_strategy: search.telemetry.retrieval_strategy,
- search_latency_ms: searchLatencyMs,
- generation_latency_ms: 0,
- total_latency_ms: answer.latencyTimings?.total_latency_ms ?? Date.now() - startedAt,
- evidence_summary: answer.evidenceSummary,
- },
- });
+ answer.relevance = relevance;
+ answer.smartPanel = answer.smartPanel ? { ...answer.smartPanel, relevance } : answer.smartPanel;
+ answer.smartApiPlan = smartApiPlan;
+ answer.scoreExplanations = answerScoreExplanations;
+
+ if (args.logQuery !== false)
+ await logRagQuery({
+ owner_id: args.ownerId ?? null,
+ query: args.query,
+ answer: answer.answer,
+ source_chunk_ids: answerInputResults.map((result) => result.id),
+ model: null,
+ metadata: {
+ document_id: args.documentId ?? null,
+ document_ids: args.documentIds ?? null,
+ grounded: answer.grounded,
+ confidence: answer.confidence,
+ routing_mode: answer.routingMode,
+ routing_reason: answer.routingReason,
+ query_class: queryClass,
+ fallback_reason: fallbackReasonFromRouting(answer.routingReason),
+ model_used: null,
+ retrieved_candidate_count: results.length,
+ ...smartApiLogMetadata(smartApiPlan),
+ ...answerRankMetadata,
+ ...memoryLogMetadata,
+ ...scoreLogMetadata,
+ cited_chunk_count: answer.citations.length,
+ quote_count: answer.quoteCards?.length ?? 0,
+ visual_evidence_count: answer.visualEvidence?.length ?? 0,
+ related_document_count: relatedDocuments.length,
+ search_cache_hit: search.telemetry.search_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ retrieval_strategy: search.telemetry.retrieval_strategy,
+ weighted_top_score: search.telemetry.weighted_top_score,
+ rrf_top_score: search.telemetry.rrf_top_score,
+ search_latency_ms: searchLatencyMs,
+ generation_latency_ms: 0,
+ total_latency_ms: answer.latencyTimings?.total_latency_ms ?? Date.now() - startedAt,
+ evidence_summary: answer.evidenceSummary,
+ source_coverage: answer.sourceCoverage,
+ },
+ });
setCachedAnswer(args, answer);
return answer;
@@ -935,26 +2178,42 @@ export async function answerQuestionWithScope(args: {
Rules:
- Answer directly from the provided excerpts only.
-- Be concise: usually 1-2 short paragraphs or 4-8 bullets unless source complexity requires more.
-- Prefer concise, high-yield clinical bullets. Avoid generic background.
+- Use model-generated clinical synthesis by default; do not stitch disconnected source quotes into the answer.
+- Format the answer as concise clinical sections when supported: Bottom line, Required actions, Monitoring/timing, Medication/dose details, Escalation/risk, Documentation/forms, Source gaps.
+- Omit sections that are not supported by the retrieved excerpts.
+- Be concise: usually 3-6 high-yield bullets total and about 120-180 words unless source complexity requires more.
- Prefer Australian or WA-specific guidance when present in the sources.
- Do not provide patient-specific medical advice.
- If the excerpts do not support a direct answer, say that the uploaded documents do not contain enough information.
-- Use wording close to the retrieved source text.
+- Use practical clinical wording, but keep every claim tied to retrieved source content.
- Put the grounded synthesis first. Then include supported detail sections only when they add clinically useful detail.
- Compare sources when several documents are relevant. Mention gaps or weak support when the evidence is narrow.
- Include clinically practical details and caveats only when supported.
- Use only the strongest 3-5 citations, not every source.
-- Bold only source-supported clinical specifics using **bold**: medications, thresholds, timing, escalation triggers, required actions, contraindications.
+- Sources are ordered by answer relevance. Prioritize earlier sources unless a later source directly resolves a conflict or gap.
+- When sources come from multiple documents, synthesize by clinical theme/action. Do not list each document separately unless the question asks for a comparison.
+- For multi-document answers, merge overlapping guidance once, then call out document-specific differences, conflicts, or gaps only when supported.
+- Keep multi-document answers fast and focused: use the fused source brief and balanced source guide to cite at least two documents when the answer combines them.
+- Treat the fused source brief as an orientation layer only. Verify every claim against the raw source excerpts below it.
+- Structured memory lines are indexing-time source facts mapped back to source chunks. Use them to focus the answer, but cite the original chunks.
+- Start with the direct answer. Omit tangential background even when it appears in retrieved sources.
+- Bold only source-supported high-yield details using **bold**: medications, thresholds, timing, escalation triggers, required actions, contraindications, and terms central to the question.
+- Do not bold whole sentences or routine filler wording.
- Do not use Markdown other than **bold** inside answer or answerSections.
-- Include 2-4 short exact quotes in quoteCards; quotes must be copied from the retrieved source excerpts.
+ - Include 1-3 short exact quotes in quoteCards; quotes must be copied from the retrieved source excerpts.
+ - Do not insert JSON-like fragments, key-value dumps, or objects in heading or body fields. Do not output strings containing keys such as answer, heading, citation_chunk_ids, or raw braces.
+- If a heading/body would include key-value pairs or JSON-like syntax, omit that section or return only concise natural language text.
- Return data matching the supplied structured output schema.`;
function buildAnswerInput(contextResults: SearchResult[]) {
+ const sourceGuide = crossDocumentPlan.enabled ? buildCrossDocumentSourceGuide(contextResults) : "";
+ const fusedBrief = crossDocumentFusionBrief?.text ?? "";
+ const crossDocumentContext = [sourceGuide, fusedBrief].filter(Boolean).join("\n\n");
return `Question:
${args.query}
Sources:
+${crossDocumentContext ? `${crossDocumentContext}\n\n` : ""}
${buildRagSourceBlock(contextResults)}`;
}
@@ -964,6 +2223,7 @@ ${buildRagSourceBlock(contextResults)}`;
let retriedWithStrong = false;
let openAIUsage: OpenAITokenUsage = {};
const openAIRequestIds: string[] = [];
+ let contextPackLatencyMs = 0;
async function generateWithModel(model: string, contextResults: SearchResult[]): Promise {
const input = buildAnswerInput(contextResults);
@@ -975,9 +2235,11 @@ ${buildRagSourceBlock(contextResults)}`;
operation: "answer",
schemaName: "clinical_rag_answer",
instructions: answerInstructions,
- promptCacheKey: "clinical-rag-answer-v2",
+ promptCacheKey: "clinical-rag-answer-v5",
reasoningEffort:
- model === env.OPENAI_STRONG_ANSWER_MODEL ? env.OPENAI_STRONG_REASONING_EFFORT : env.OPENAI_FAST_REASONING_EFFORT,
+ model === env.OPENAI_STRONG_ANSWER_MODEL
+ ? env.OPENAI_STRONG_REASONING_EFFORT
+ : env.OPENAI_FAST_REASONING_EFFORT,
});
openAIUsage = addOpenAIUsage(openAIUsage, result.usage);
if (result.requestId) openAIRequestIds.push(result.requestId);
@@ -987,82 +2249,111 @@ ${buildRagSourceBlock(contextResults)}`;
}
}
- const fastContextResults = route.mode === "fast" ? results.slice(0, 4) : results;
- await args.onProgress?.({
- stage: "generating",
- message: `Generating cited answer with ${route.mode} route.`,
- mode: route.mode,
- model: route.model,
- reason: route.reason,
- });
- let generated = await generateWithModel(route.model!, fastContextResults);
- let answer = parseAnswerJson(generated.text, fastContextResults);
- if (shouldRetryWithStrongAfterFast({ route, answer, results })) {
- modelUsed = env.OPENAI_STRONG_ANSWER_MODEL;
- routingReason = `${route.reason}; fast_unsupported_retry_strong`;
- retriedWithStrong = true;
+ function summarizeGenerationFailureReason(error: unknown) {
+ if (error instanceof Error && error.message.trim()) return error.message.trim();
+ if (typeof error === "string" && error.trim()) return error.trim();
+ return "generation encountered an error";
+ }
+
+ async function buildGenerationFallbackAnswer(
+ error: unknown,
+ relatedDocuments: RelatedDocument[],
+ ): Promise {
+ const hasSources = answerInputResults.length > 0;
+ const fallbackCitations = compactCitations(answerInputResults);
+ const sanitizedReason = summarizeGenerationFailureReason(error);
+ const fallbackBestSource = hasSources ? (selectBestSourceRecommendation(answerInputResults, quoteCards) ?? bestSource) : null;
+ const fallbackSmartPanel = hasSources
+ ? { ...smartPanel, relevance, bestSource: fallbackBestSource, relatedDocuments }
+ : { ...emptyPanel, relevance, relatedDocuments };
+
+ return {
+ answer: boldHighYieldClinicalText(
+ hasSources
+ ? "I found matching indexed passages, but could not generate a finalized answer right now. Review the source snippets below."
+ : "I could not find enough indexed support in the available documents to answer this query yet.",
+ args.query,
+ ),
+ grounded: false,
+ confidence: hasSources ? deriveConfidence(answerInputResults, fallbackCitations.length) : "unsupported",
+ citations: hasSources ? fallbackCitations : [],
+ sources: answerInputResults,
+ modelUsed: null,
+ routingMode: "unsupported",
+ routingReason: `${route.reason}; generation_fallback:${sanitizedReason}`,
+ queryClass,
+ latencyTimings: {
+ search_cache_hit: search.telemetry.search_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ context_pack_latency_ms: contextPackLatencyMs,
+ search_latency_ms: searchLatencyMs,
+ generation_latency_ms: generationLatencyMs,
+ total_latency_ms: Date.now() - startedAt,
+ },
+ answerSections: [],
+ quoteCards: hasSources ? reconcileQuoteCards(quoteCards, answerInputResults, args.query) : [],
+ visualEvidence: hasSources ? visualEvidence : [],
+ bestSource: hasSources ? fallbackBestSource : null,
+ documentBreakdown: hasSources ? documentBreakdown : [],
+ evidenceSummary: hasSources ? evidenceSummary : emptyPanel.evidenceSummary,
+ sourceCoverage: hasSources ? sourceCoverage : emptyPanel.sourceCoverage,
+ conflictsOrGaps: hasSources ? conflictsOrGaps : [],
+ smartPanel: fallbackSmartPanel,
+ relatedDocuments,
+ relevance,
+ memoryCardsUsed: hasSources ? memoryCardsUsed : [],
+ indexingVersion: ragDeepMemoryVersion,
+ indexingQuality,
+ smartApiPlan: buildCurrentSmartApiPlan("unsupported", `${route.reason}; generation_fallback`),
+ scoreExplanations: answerScoreExplanations,
+ } satisfies RagAnswer;
+ }
+
+ const fastContextResults =
+ route.mode === "fast" && !crossDocumentPlan.enabled ? answerInputResults.slice(0, 4) : answerInputResults;
+ try {
await args.onProgress?.({
- stage: "retrying",
- message: "Fast answer was unsupported, retrying with the strong model.",
- mode: "strong",
- model: env.OPENAI_STRONG_ANSWER_MODEL,
- reason: routingReason,
+ stage: "generating",
+ message: `Generating cited answer with ${route.mode} route.`,
+ mode: route.mode,
+ model: route.model,
+ reason: route.reason,
});
- generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, results);
- answer = parseAnswerJson(generated.text, results);
- }
- await args.onProgress?.({ stage: "finalizing", message: "Checking citations and source metadata." });
-
- const relatedDocuments = await relatedDocumentsPromise;
- answer.sources = results;
- answer.quoteCards = reconcileQuoteCards(answer.quoteCards, results, args.query);
- answer.documentBreakdown = documentBreakdown;
- answer.evidenceSummary = evidenceSummary;
- answer.sourceCoverage = sourceCoverage;
- answer.conflictsOrGaps = answer.conflictsOrGaps?.length ? answer.conflictsOrGaps : conflictsOrGaps;
- answer.visualEvidence = visualEvidence;
- answer.bestSource = selectBestSourceRecommendation(results, answer.quoteCards) ?? bestSource;
- answer.relatedDocuments = relatedDocuments;
- answer.smartPanel = { ...smartPanel, bestSource: answer.bestSource, relatedDocuments };
- answer.modelUsed = modelUsed;
- answer.routingMode = retriedWithStrong ? "strong" : route.mode;
- answer.routingReason = routingReason;
- answer.openAIRequestIds = openAIRequestIds;
- answer.openAIUsage = hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined;
- answer.latencyTimings = {
- search_cache_hit: search.telemetry.search_cache_hit,
- text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
- embedding_skipped: search.telemetry.embedding_skipped,
- embedding_latency_ms: search.telemetry.embedding_latency_ms,
- embedding_cache_hit: search.telemetry.embedding_cache_hit,
- supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
- rerank_latency_ms: search.telemetry.rerank_latency_ms,
- search_latency_ms: searchLatencyMs,
- generation_latency_ms: generationLatencyMs,
- total_latency_ms: Date.now() - startedAt,
- };
+ const contextPackStartedAt = Date.now();
+ let packedContextResults = await packAdjacentSourceContext(createAdminClient(), fastContextResults, queryClass, {
+ crossDocument: crossDocumentPlan.enabled,
+ });
+ contextPackLatencyMs += Date.now() - contextPackStartedAt;
+ let generated = await generateWithModel(route.model!, packedContextResults);
+ let answer = parseAnswerJson(generated.text, packedContextResults, args.query);
+ if (shouldRetryWithStrongAfterFast({ route, answer, results: answerInputResults })) {
+ modelUsed = env.OPENAI_STRONG_ANSWER_MODEL;
+ routingReason = `${route.reason}; fast_unsupported_retry_strong`;
+ retriedWithStrong = true;
+ await args.onProgress?.({
+ stage: "retrying",
+ message: "Fast answer was unsupported, retrying with the strong model.",
+ mode: "strong",
+ model: env.OPENAI_STRONG_ANSWER_MODEL,
+ reason: routingReason,
+ });
+ const retryContextPackStartedAt = Date.now();
+ packedContextResults = await packAdjacentSourceContext(createAdminClient(), answerInputResults, queryClass, {
+ crossDocument: crossDocumentPlan.enabled,
+ });
+ contextPackLatencyMs += Date.now() - retryContextPackStartedAt;
+ generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults);
+ answer = parseAnswerJson(generated.text, packedContextResults, args.query);
+ }
+ await args.onProgress?.({ stage: "finalizing", message: "Checking citations and source metadata." });
- if (args.logQuery !== false) await logRagQuery({
- owner_id: args.ownerId ?? null,
- query: args.query,
- answer: answer.answer,
- source_chunk_ids: results.map((result) => result.id),
- model: modelUsed,
- metadata: {
- document_id: args.documentId ?? null,
- document_ids: args.documentIds ?? null,
- grounded: answer.grounded,
- confidence: answer.confidence,
- routing_mode: answer.routingMode,
- routing_reason: routingReason,
- model_used: modelUsed,
- fast_model: env.OPENAI_FAST_ANSWER_MODEL,
- strong_model: env.OPENAI_STRONG_ANSWER_MODEL,
- retrieved_candidate_count: results.length,
- cited_chunk_count: answer.citations.length,
- quote_count: answer.quoteCards?.length ?? 0,
- visual_evidence_count: answer.visualEvidence?.length ?? 0,
- related_document_count: relatedDocuments.length,
+ const relatedDocuments = await relatedDocumentsPromise;
+ const answerTimings = {
search_cache_hit: search.telemetry.search_cache_hit,
text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
embedding_skipped: search.telemetry.embedding_skipped,
@@ -1070,18 +2361,170 @@ ${buildRagSourceBlock(contextResults)}`;
embedding_cache_hit: search.telemetry.embedding_cache_hit,
supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
rerank_latency_ms: search.telemetry.rerank_latency_ms,
- retrieval_strategy: search.telemetry.retrieval_strategy,
+ context_pack_latency_ms: contextPackLatencyMs,
search_latency_ms: searchLatencyMs,
generation_latency_ms: generationLatencyMs,
- total_latency_ms: answer.latencyTimings.total_latency_ms,
- openai_request_ids: openAIRequestIds,
- openai_usage: answer.openAIUsage ?? null,
- evidence_summary: answer.evidenceSummary,
- },
- });
+ total_latency_ms: Date.now() - startedAt,
+ };
- setCachedAnswer(args, answer);
- return answer;
+ if (answer.citations.length > 0 && isUnusableGeneratedAnswer(answer)) {
+ answer = buildExtractiveAnswer({
+ query: args.query,
+ queryClass,
+ results: answerInputResults,
+ quoteCards,
+ documentBreakdown,
+ evidenceSummary,
+ sourceCoverage,
+ conflictsOrGaps,
+ visualEvidence,
+ bestSource,
+ smartPanel: { ...smartPanel, relevance, bestSource, relatedDocuments },
+ relatedDocuments,
+ routeReason: `${routingReason}; structured_output_fallback`,
+ timings: answerTimings,
+ });
+ answer.modelUsed = modelUsed;
+ } else {
+ answer = boldRagAnswerHighYieldText(answer, args.query);
+ answer.sources = answerInputResults;
+ answer.quoteCards = reconcileQuoteCards(answer.quoteCards, answerInputResults, args.query);
+ answer.documentBreakdown = documentBreakdown;
+ answer.evidenceSummary = evidenceSummary;
+ answer.sourceCoverage = sourceCoverage;
+ answer.conflictsOrGaps = answer.conflictsOrGaps?.length ? answer.conflictsOrGaps : conflictsOrGaps;
+ answer.visualEvidence = visualEvidence;
+ answer.bestSource = selectBestSourceRecommendation(answerInputResults, answer.quoteCards) ?? bestSource;
+ answer.relatedDocuments = relatedDocuments;
+ answer.smartPanel = { ...smartPanel, relevance, bestSource: answer.bestSource, relatedDocuments };
+ answer.routingMode = retriedWithStrong ? "strong" : route.mode;
+ answer.routingReason = routingReason;
+ }
+ answer.modelUsed = modelUsed;
+ answer.queryClass = queryClass;
+ answer.openAIRequestIds = openAIRequestIds;
+ answer.openAIUsage = hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined;
+ answer.latencyTimings = answerTimings;
+ answer.memoryCardsUsed = memoryCardsUsed;
+ answer.indexingVersion = ragDeepMemoryVersion;
+ answer.indexingQuality = indexingQuality;
+ answer.smartApiPlan = buildCurrentSmartApiPlan(answer.routingMode, answer.routingReason);
+ answer.scoreExplanations = answerScoreExplanations;
+ answer.relevance = relevance;
+ answer.smartPanel = answer.smartPanel ? { ...answer.smartPanel, relevance } : answer.smartPanel;
+
+ if (args.logQuery !== false)
+ await logRagQuery({
+ owner_id: args.ownerId ?? null,
+ query: args.query,
+ answer: answer.answer,
+ source_chunk_ids: answerInputResults.map((result) => result.id),
+ model: modelUsed,
+ metadata: {
+ document_id: args.documentId ?? null,
+ document_ids: args.documentIds ?? null,
+ grounded: answer.grounded,
+ confidence: answer.confidence,
+ routing_mode: answer.routingMode,
+ routing_reason: routingReason,
+ query_class: queryClass,
+ fallback_reason: fallbackReasonFromRouting(answer.routingReason),
+ model_used: modelUsed,
+ fast_model: env.OPENAI_FAST_ANSWER_MODEL,
+ strong_model: env.OPENAI_STRONG_ANSWER_MODEL,
+ retrieved_candidate_count: results.length,
+ ...smartApiLogMetadata(answer.smartApiPlan),
+ ...answerRankMetadata,
+ ...memoryLogMetadata,
+ ...scoreLogMetadata,
+ cited_chunk_count: answer.citations.length,
+ quote_count: answer.quoteCards?.length ?? 0,
+ visual_evidence_count: answer.visualEvidence?.length ?? 0,
+ related_document_count: relatedDocuments.length,
+ search_cache_hit: search.telemetry.search_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ context_pack_latency_ms: contextPackLatencyMs,
+ retrieval_strategy: search.telemetry.retrieval_strategy,
+ weighted_top_score: search.telemetry.weighted_top_score,
+ rrf_top_score: search.telemetry.rrf_top_score,
+ search_latency_ms: searchLatencyMs,
+ generation_latency_ms: generationLatencyMs,
+ total_latency_ms: answer.latencyTimings.total_latency_ms,
+ openai_request_ids: openAIRequestIds,
+ openai_usage: answer.openAIUsage ?? null,
+ evidence_summary: answer.evidenceSummary,
+ source_coverage: answer.sourceCoverage,
+ },
+ });
+
+ setCachedAnswer(args, answer);
+ return answer;
+ } catch (error) {
+ const relatedDocuments = await relatedDocumentsPromise;
+ await args.onProgress?.({
+ stage: "finalizing",
+ message: "Generation failed, returning source-based fallback answer.",
+ mode: "unsupported",
+ reason: "generation_fallback",
+ });
+ const fallbackAnswer = await buildGenerationFallbackAnswer(error, relatedDocuments);
+ if (args.logQuery !== false)
+ await logRagQuery({
+ owner_id: args.ownerId ?? null,
+ query: args.query,
+ answer: fallbackAnswer.answer,
+ source_chunk_ids: answerInputResults.map((result) => result.id),
+ model: null,
+ metadata: {
+ document_id: args.documentId ?? null,
+ document_ids: args.documentIds ?? null,
+ grounded: fallbackAnswer.grounded,
+ confidence: fallbackAnswer.confidence,
+ routing_mode: fallbackAnswer.routingMode,
+ routing_reason: fallbackAnswer.routingReason,
+ query_class: queryClass,
+ fallback_reason: fallbackReasonFromRouting(fallbackAnswer.routingReason),
+ model_used: null,
+ fast_model: env.OPENAI_FAST_ANSWER_MODEL,
+ strong_model: env.OPENAI_STRONG_ANSWER_MODEL,
+ retrieved_candidate_count: results.length,
+ ...(fallbackAnswer.smartApiPlan ? smartApiLogMetadata(fallbackAnswer.smartApiPlan) : {}),
+ ...answerRankMetadata,
+ ...memoryLogMetadata,
+ ...scoreLogMetadata,
+ cited_chunk_count: fallbackAnswer.citations.length,
+ quote_count: fallbackAnswer.quoteCards?.length ?? 0,
+ visual_evidence_count: fallbackAnswer.visualEvidence?.length ?? 0,
+ related_document_count: relatedDocuments.length,
+ search_cache_hit: search.telemetry.search_cache_hit,
+ text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms,
+ embedding_skipped: search.telemetry.embedding_skipped,
+ embedding_latency_ms: search.telemetry.embedding_latency_ms,
+ embedding_cache_hit: search.telemetry.embedding_cache_hit,
+ supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms,
+ rerank_latency_ms: search.telemetry.rerank_latency_ms,
+ context_pack_latency_ms: contextPackLatencyMs,
+ retrieval_strategy: "generation_fallback",
+ weighted_top_score: search.telemetry.weighted_top_score,
+ rrf_top_score: search.telemetry.rrf_top_score,
+ search_latency_ms: searchLatencyMs,
+ generation_latency_ms: generationLatencyMs,
+ total_latency_ms: fallbackAnswer.latencyTimings?.total_latency_ms ?? Date.now() - startedAt,
+ openai_request_ids: fallbackAnswer.openAIRequestIds,
+ openai_usage: fallbackAnswer.openAIUsage,
+ evidence_summary: fallbackAnswer.evidenceSummary,
+ source_coverage: fallbackAnswer.sourceCoverage,
+ },
+ });
+
+ setCachedAnswer(args, fallbackAnswer);
+ return fallbackAnswer;
+ }
}
export async function summarizeDocument(documentId: string, ownerId?: string) {
@@ -1125,7 +2568,7 @@ export async function summarizeDocument(documentId: string, ownerId?: string) {
})) as SearchResult[];
const summaryInstructions = `Summarize a clinical document for practical psychiatric use in Perth, Australia.
-Use only the excerpts provided. Focus on high-yield actions, thresholds, medication or risk monitoring, exceptions, and citations.
+Use only the excerpts provided. Start directly with the most useful clinical point; do not prefix the answer with "Summary", "Key practical points", or similar labels. Focus on high-yield actions, thresholds, medication or risk monitoring, exceptions, and citations. Exclude administrative document-control details unless they change clinical action.
Return data matching the supplied structured output schema.`;
const summaryInput = `Document:
${document.title}
@@ -1142,7 +2585,8 @@ ${buildRagSourceBlock(results)}`;
promptCacheKey: "clinical-document-summary-v1",
reasoningEffort: env.OPENAI_SUMMARY_REASONING_EFFORT,
});
- const answer = parseAnswerJson(generated.text, results);
+ const answer = parseAnswerJson(generated.text, results, "summary");
+ answer.answer = cleanClinicalSummaryText(answer.answer);
answer.quoteCards = reconcileQuoteCards(answer.quoteCards, results, "summary");
answer.documentBreakdown = buildDocumentBreakdown(results, answer.quoteCards);
answer.evidenceSummary = buildEvidenceSummary(results, answer.quoteCards);
diff --git a/src/lib/smart-rag-api.ts b/src/lib/smart-rag-api.ts
new file mode 100644
index 000000000..f6ab909ef
--- /dev/null
+++ b/src/lib/smart-rag-api.ts
@@ -0,0 +1,171 @@
+import { citationFromResult, documentCitationHref, formatCompactCitationLabel } from "@/lib/citations";
+import { sourceStrengthForSimilarity } from "@/lib/evidence";
+import { sourceTextForDisplay } from "@/lib/source-text-sanitizer";
+import type { RagAnswer, RagQueryClass, SearchResult, SmartRagApiPlan, SmartRagSourceLink } from "@/lib/types";
+
+type RetrievalStrategy = SmartRagApiPlan["retrievalStrategy"];
+
+type BuildSmartRagApiPlanArgs = {
+ query: string;
+ queryClass: RagQueryClass;
+ results: SearchResult[];
+ routeMode?: RagAnswer["routingMode"];
+ routeReason?: string;
+ retrievalStrategy?: RetrievalStrategy;
+ maxLinks?: number;
+ preferredResponseMode?: SmartRagApiPlan["responseMode"];
+};
+
+const crossDocumentPattern =
+ /\b(?:across|combine|combined|synthesi[sz]e|together|overall|all documents|these documents|different documents|multiple documents|compare|versus|vs|between)\b/i;
+
+const queryClassIntent: Record = {
+ document_lookup: "find_document",
+ table_threshold: "find_threshold_or_table",
+ medication_dose_risk: "medication_or_risk_answer",
+ comparison: "compare_sources",
+ broad_summary: "summarize_sources",
+ unsupported_or_general: "general_or_unsupported",
+};
+
+function compact(value: string, limit: number) {
+ const normalized = sourceTextForDisplay(value).replace(/\s+/g, " ").trim();
+ if (normalized.length <= limit) return normalized;
+ return `${normalized.slice(0, limit - 3).trim()}...`;
+}
+
+function uniqueDocumentCount(results: SearchResult[]) {
+ return new Set(results.map((result) => result.document_id)).size;
+}
+
+function resultScore(result: SearchResult) {
+ return result.hybrid_score ?? result.similarity ?? 0;
+}
+
+function linkReason(result: SearchResult, queryClass: RagQueryClass) {
+ if (queryClass === "document_lookup") return "Best document/title match";
+ if (queryClass === "table_threshold") return "Relevant table, threshold, or monitoring evidence";
+ if (queryClass === "medication_dose_risk") return "Medication, dose, monitoring, or risk evidence";
+ if (queryClass === "comparison") return "Comparison source";
+ if (queryClass === "broad_summary") return "Summary source";
+ if (result.memory_cards?.length) return "Structured source memory match";
+ return "Relevant indexed source";
+}
+
+function buildCoreSourceLinks(
+ results: SearchResult[],
+ queryClass: RagQueryClass,
+ maxLinks: number,
+): SmartRagSourceLink[] {
+ const seen = new Set();
+ const links: SmartRagSourceLink[] = [];
+ const ranked = [...results].sort((a, b) => resultScore(b) - resultScore(a) || a.chunk_index - b.chunk_index);
+
+ for (const result of ranked) {
+ const citation = citationFromResult(result);
+ const key = `${citation.document_id}:${citation.page_number}:${citation.chunk_id}`;
+ if (seen.has(key)) continue;
+ seen.add(key);
+
+ links.push({
+ id: key,
+ label: formatCompactCitationLabel(citation),
+ href: documentCitationHref(citation),
+ document_id: citation.document_id,
+ chunk_id: citation.chunk_id,
+ title: citation.title,
+ file_name: citation.file_name,
+ page_number: citation.page_number,
+ source_strength: result.source_strength ?? sourceStrengthForSimilarity(result.similarity),
+ reason: linkReason(result, queryClass),
+ snippet: compact(`${result.section_heading ? `${result.section_heading}: ` : ""}${result.content}`, 220),
+ });
+
+ if (links.length >= maxLinks) break;
+ }
+
+ return links;
+}
+
+function responseMode(
+ args: Pick,
+) {
+ if (args.results.length === 0 || args.routeMode === "unsupported") return "unsupported";
+ if (args.preferredResponseMode) return args.preferredResponseMode;
+ if (args.queryClass === "document_lookup") return "document_lookup";
+ if (args.routeMode === "extractive") return "extractive_answer";
+ if (args.routeMode === "strong") return "strong_synthesis";
+ if (
+ args.queryClass === "comparison" ||
+ (uniqueDocumentCount(args.results) > 1 && (args.queryClass === "broad_summary" || crossDocumentPattern.test(args.query)))
+ ) {
+ return "multi_document_synthesis";
+ }
+ return "fast_grounded_answer";
+}
+
+function latencyPlan(
+ mode: SmartRagApiPlan["responseMode"],
+ retrievalStrategy: SmartRagApiPlan["retrievalStrategy"],
+): SmartRagApiPlan["latencyPlan"] {
+ if (mode === "unsupported") return "no_supported_answer";
+ if (mode === "strong_synthesis") return "strong_generation";
+ if (retrievalStrategy === "search_cache" || retrievalStrategy === "text_fast_path" || retrievalStrategy === "document_lookup_fast_path") {
+ return "cache_or_text_first";
+ }
+ return "balanced_hybrid";
+}
+
+function answerFocus(args: {
+ queryClass: RagQueryClass;
+ mode: SmartRagApiPlan["responseMode"];
+ documentCount: number;
+ linkCount: number;
+}) {
+ if (args.mode === "unsupported" || args.linkCount === 0) return "Report that the indexed sources do not support a reliable answer.";
+ if (args.mode === "document_lookup") return "Open with the best matching document and page, then show the strongest source link.";
+ if (args.mode === "multi_document_synthesis") {
+ return `Merge overlapping guidance across ${args.documentCount} documents, cite the strongest links, and call out conflicts or gaps only when supported.`;
+ }
+ if (args.queryClass === "table_threshold") return "Lead with the exact threshold, table row, or monitoring rule and cite the source section.";
+ if (args.queryClass === "medication_dose_risk") return "Lead with medication, dose, monitoring, escalation, and risk details that directly answer the question.";
+ if (args.queryClass === "broad_summary") return "Give a concise source-backed summary and avoid tangential background.";
+ return "Answer directly using the highest-ranked source links.";
+}
+
+function streamPlan(mode: SmartRagApiPlan["responseMode"], retrievalStrategy: SmartRagApiPlan["retrievalStrategy"]) {
+ if (mode === "unsupported") return ["Search indexed sources", "Report unsupported evidence", "Show nearby source links if available"];
+ if (mode === "document_lookup") return ["Match document metadata", "Rank matching pages", "Return document links"];
+ if (mode === "multi_document_synthesis") {
+ return ["Classify query intent", "Balance sources across documents", "Fuse strongest points", "Generate cited answer"];
+ }
+ if (retrievalStrategy === "text_fast_path" || retrievalStrategy === "search_cache") {
+ return ["Use fast retrieval", "Rank source evidence", "Return cited answer"];
+ }
+ return ["Run hybrid retrieval", "Rank source evidence", "Generate cited answer"];
+}
+
+export function buildSmartRagApiPlan(args: BuildSmartRagApiPlanArgs): SmartRagApiPlan {
+ const retrievalStrategy = args.retrievalStrategy ?? "unknown";
+ const mode = responseMode(args);
+ const coreSourceLinks = buildCoreSourceLinks(args.results, args.queryClass, args.maxLinks ?? 5);
+ const documentCount = uniqueDocumentCount(args.results);
+
+ return {
+ query: args.query,
+ queryClass: args.queryClass,
+ intent: queryClassIntent[args.queryClass],
+ responseMode: mode,
+ retrievalStrategy,
+ latencyPlan: latencyPlan(mode, retrievalStrategy),
+ answerFocus: answerFocus({
+ queryClass: args.queryClass,
+ mode,
+ documentCount,
+ linkCount: coreSourceLinks.length,
+ }),
+ sourceLinkCount: coreSourceLinks.length,
+ coreSourceLinks,
+ streamPlan: streamPlan(mode, retrievalStrategy),
+ };
+}
diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts
new file mode 100644
index 000000000..cd9a49bf4
--- /dev/null
+++ b/src/lib/source-text-sanitizer.ts
@@ -0,0 +1,169 @@
+const completeImageDataBlockPattern = /\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g;
+const trailingImageDataBlockPattern = /\[\[IMAGE_DATA_START\]\][\s\S]*$/g;
+const leadingImageDataBlockRemainderPattern = /^[\s\S]*?\[\[IMAGE_DATA_END\]\]/g;
+
+const internalImageMetadataPattern =
+ /\b(?:Image ID|Source kind|Image type|Table role|Clinical use class|Clinical use reason|Clinical signal score|Admin signal score|Storage path|Image path)\s*:\s*[^;|]+[;|]?\s*/gi;
+
+function compactWhitespace(value: string) {
+ return value.replace(/\s+/g, " ").trim();
+}
+
+function readableWhitespace(value: string) {
+ return value
+ .replace(/[ \t]+/g, " ")
+ .replace(/[ \t]*\n[ \t]*/g, "\n")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+}
+
+function extractImageBlockField(block: string, field: string) {
+ const pattern = new RegExp(
+ `${field}:\\s*([\\s\\S]*?)(?:;\\s*[A-Z][A-Za-z _-]{1,40}:|\\s*\\[\\[IMAGE_DATA_END\\]\\])`,
+ "i",
+ );
+ return compactWhitespace(block.match(pattern)?.[1] ?? "");
+}
+
+function readableImageBlock(block: string) {
+ const title = extractImageBlockField(block, "Table title");
+ const label = extractImageBlockField(block, "Table label");
+ const description = extractImageBlockField(block, "Description");
+ const caption = extractImageBlockField(block, "Caption");
+
+ return compactWhitespace(
+ [title ? `Clinical table: ${title}` : "", label && !title ? `Clinical table: ${label}` : "", description || caption]
+ .filter(Boolean)
+ .join(". "),
+ );
+}
+
+function tableCells(row: string) {
+ return row
+ .replace(/^\s*\|/, "")
+ .replace(/\|\s*$/, "")
+ .split("|")
+ .map((cell) => compactWhitespace(cell))
+ .filter(Boolean);
+}
+
+function isSeparatorRow(cells: string[]) {
+ return cells.length > 0 && cells.every((cell) => /^:?-{2,}:?$/.test(cell));
+}
+
+function readableTableRows(tableText: string) {
+ const normalizedRows = tableText
+ .replace(/\r?\n/g, " ")
+ .replace(/\|\s*\|\s*/g, "||")
+ .split(/\s*\|\|\s*/)
+ .map((row) => tableCells(row))
+ .filter((cells) => cells.length > 0 && !isSeparatorRow(cells));
+
+ if (normalizedRows.length === 0) return "";
+
+ const [headers, ...rows] = normalizedRows;
+ if (headers.length === 0) return "";
+ if (rows.length === 0) return headers.join(" | ");
+
+ return [
+ headers.join(" | "),
+ ...rows.slice(0, 8).map((row) => {
+ const first = row[0] ?? "Row";
+ const details = row
+ .slice(1)
+ .map((cell, index) => {
+ const header = headers[index + 1];
+ return header ? `${header}: ${cell}` : cell;
+ })
+ .join("; ");
+ return details ? `- ${first}: ${details}` : `- ${first}`;
+ }),
+ ].join("\n");
+}
+
+function readableImageBlockForViewer(block: string) {
+ const title = extractImageBlockField(block, "Table title");
+ const label = extractImageBlockField(block, "Table label");
+ const description = extractImageBlockField(block, "Description");
+ const caption = extractImageBlockField(block, "Caption");
+ const tableText = extractImageBlockField(block, "Table text");
+ const intro = compactWhitespace([title || label ? `Clinical table: ${title || label}` : "", description || caption].filter(Boolean).join(". "));
+ const table = readableTableRows(tableText);
+
+ return readableWhitespace([intro, table].filter(Boolean).join("\n\n"));
+}
+
+export function stripInternalImageDataBlocks(text: string) {
+ return compactWhitespace(
+ text
+ .replace(completeImageDataBlockPattern, " ")
+ .replace(trailingImageDataBlockPattern, " ")
+ .replace(leadingImageDataBlockRemainderPattern, " ")
+ .replace(internalImageMetadataPattern, " "),
+ );
+}
+
+export function sourceTextForModel(text: string) {
+ return compactWhitespace(
+ text
+ .replace(completeImageDataBlockPattern, (block) => readableImageBlock(block))
+ .replace(trailingImageDataBlockPattern, " ")
+ .replace(leadingImageDataBlockRemainderPattern, " ")
+ .replace(internalImageMetadataPattern, " "),
+ );
+}
+
+export function sourceTextForDisplay(text: string) {
+ return stripInternalImageDataBlocks(text);
+}
+
+export function sourceTextForDisplayPreservingBreaks(text: string) {
+ return readableWhitespace(
+ text
+ .replace(completeImageDataBlockPattern, " ")
+ .replace(trailingImageDataBlockPattern, " ")
+ .replace(leadingImageDataBlockRemainderPattern, " ")
+ .replace(internalImageMetadataPattern, " "),
+ );
+}
+
+export function sourceTextForDocumentViewer(text: string) {
+ return readableWhitespace(
+ text
+ .replace(completeImageDataBlockPattern, (block) => readableImageBlockForViewer(block))
+ .replace(trailingImageDataBlockPattern, " ")
+ .replace(leadingImageDataBlockRemainderPattern, " ")
+ .replace(internalImageMetadataPattern, " "),
+ );
+}
+
+export function sourceTextForIndexedPage(text: string) {
+ return text
+ .replace(/\r/g, "\n")
+ .replace(completeImageDataBlockPattern, (block) => readableImageBlockForViewer(block))
+ .replace(trailingImageDataBlockPattern, " ")
+ .replace(leadingImageDataBlockRemainderPattern, " ")
+ .replace(internalImageMetadataPattern, " ")
+ .replace(/[ \t]+$/gm, "")
+ .replace(/\n{4,}/g, "\n\n\n")
+ .trim();
+}
+
+export function cleanClinicalSummaryText(text: string) {
+ const cleaned = sourceTextForDisplayPreservingBreaks(text)
+ .replace(/\bSource mentions:\s*/gi, "")
+ .replace(
+ /(?:\s+-\s*)?(?:Medication point|Table evidence|Threshold\/action|Risk\/escalation|Workflow step|Section summary|Source point|Monitoring)\s*:\s*/gi,
+ ". ",
+ )
+ .replace(/^\s*(?:[-•]\s+|\*\s+)+/, "")
+ .replace(
+ /^(?:key\s+(?:practical|clinical|source-backed)\s+points?|high-yield\s+points?|practical\s+points?|clinical\s+summary|document\s+summary|source-backed\s+summary|summary|bottom\s+line|answer)\s*[:\-]\s*/i,
+ "",
+ )
+ .replace(/^\s*[.;:\-]\s*/, "")
+ .replace(/\s+([,.;:])/g, "$1")
+ .replace(/(?:\.\s*){2,}/g, ". ");
+
+ return readableWhitespace(cleaned);
+}
diff --git a/src/lib/supabase/auth.ts b/src/lib/supabase/auth.ts
index 07661bd05..06b16c897 100644
--- a/src/lib/supabase/auth.ts
+++ b/src/lib/supabase/auth.ts
@@ -1,5 +1,7 @@
import { NextResponse } from "next/server";
import { createAdminClient } from "@/lib/supabase/admin";
+import { env, isLocalNoAuthMode } from "@/lib/env";
+import { isSafeLocalProjectRequest } from "@/lib/local-project-guard";
type AdminClient = ReturnType;
@@ -7,18 +9,51 @@ export type AuthenticatedUser = {
id: string;
};
+const LOCAL_OWNER_CACHE_TTL_MS = 5 * 60_000;
+
+type LocalOwnerResolutionState = {
+ cache: {
+ cacheKey: string;
+ expiresAt: number;
+ user: AuthenticatedUser;
+ } | null;
+ inFlight: {
+ cacheKey: string;
+ promise: Promise;
+ } | null;
+};
+
+type GlobalWithLocalOwnerResolutionState = typeof globalThis & {
+ __clinicalKbLocalOwnerResolutionState?: LocalOwnerResolutionState;
+};
+
+const localOwnerResolutionState = ((
+ globalThis as GlobalWithLocalOwnerResolutionState
+).__clinicalKbLocalOwnerResolutionState ??= {
+ cache: null,
+ inFlight: null,
+});
+
export class AuthenticationError extends Error {
- constructor() {
- super("Authentication required.");
+ constructor(message = "Authentication required.") {
+ super(message);
this.name = "AuthenticationError";
}
}
-export function unauthorizedResponse() {
+export function unauthorizedResponse(error?: AuthenticationError) {
+ void error;
return NextResponse.json({ error: "Authentication required." }, { status: 401 });
}
export async function requireAuthenticatedUser(request: Request, supabase: AdminClient): Promise {
+ if (isLocalNoAuthMode()) {
+ if (!isSafeLocalProjectRequest(request)) {
+ throw new AuthenticationError("Use the ensured Clinical KB local URL before calling private APIs.");
+ }
+ return resolveLocalNoAuthUser(supabase);
+ }
+
const authorization = request.headers.get("authorization") ?? "";
const match = authorization.match(/^Bearer\s+(.+)$/i);
const token = match?.[1]?.trim();
@@ -36,3 +71,108 @@ export async function requireAuthenticatedUser(request: Request, supabase: Admin
return { id: userId };
}
+
+async function resolveLocalNoAuthUser(supabase: AdminClient): Promise {
+ const configuredOwnerId = env.LOCAL_NO_AUTH_OWNER_ID?.trim();
+ if (configuredOwnerId) {
+ if (!isUuid(configuredOwnerId)) {
+ throw new Error("LOCAL_NO_AUTH_OWNER_ID must be a valid UUID.");
+ }
+ return { id: configuredOwnerId };
+ }
+
+ const configuredOwnerEmail = env.LOCAL_NO_AUTH_OWNER_EMAIL?.trim();
+ const cacheKey = `email:${configuredOwnerEmail?.toLowerCase() ?? ""}:documents-fallback`;
+ const now = Date.now();
+
+ if (localOwnerResolutionState.cache?.cacheKey === cacheKey && localOwnerResolutionState.cache.expiresAt > now) {
+ return localOwnerResolutionState.cache.user;
+ }
+
+ if (localOwnerResolutionState.inFlight?.cacheKey === cacheKey) {
+ return localOwnerResolutionState.inFlight.promise;
+ }
+
+ const promise = resolveLocalNoAuthOwnerId(supabase, configuredOwnerEmail).then((ownerId) => {
+ const user = { id: ownerId };
+ localOwnerResolutionState.cache = {
+ cacheKey,
+ expiresAt: Date.now() + LOCAL_OWNER_CACHE_TTL_MS,
+ user,
+ };
+ return user;
+ });
+
+ localOwnerResolutionState.inFlight = { cacheKey, promise };
+
+ try {
+ return await promise;
+ } finally {
+ if (localOwnerResolutionState.inFlight?.promise === promise) {
+ localOwnerResolutionState.inFlight = null;
+ }
+ }
+}
+
+async function resolveLocalNoAuthOwnerId(supabase: AdminClient, configuredOwnerEmail?: string) {
+ const fallbackOwnerId = await resolveOwnerFromDocuments(supabase);
+ if (fallbackOwnerId) return fallbackOwnerId;
+
+ const ownerIdFromEmail = await resolveOwnerByEmail(supabase, configuredOwnerEmail);
+ if (ownerIdFromEmail) return ownerIdFromEmail;
+
+ throw new Error(
+ "Local no-auth mode is enabled, but no owner could be resolved. Set LOCAL_NO_AUTH_OWNER_ID or " +
+ "LOCAL_NO_AUTH_OWNER_EMAIL, or ensure the documents table has at least one row with owner_id.",
+ );
+}
+
+function isUuid(value: string) {
+ return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
+}
+
+async function resolveOwnerByEmail(supabase: AdminClient, ownerEmail?: string) {
+ if (!ownerEmail) return null;
+
+ const normalizedEmail = ownerEmail.trim().toLowerCase();
+ if (!normalizedEmail) return null;
+
+ let page = 1;
+ const seenPages = new Set();
+
+ while (page > 0) {
+ if (seenPages.has(page)) {
+ throw new Error("Failed to resolve owner by email because the admin user listing returned a pagination loop.");
+ }
+ seenPages.add(page);
+
+ const { data, error } = await supabase.auth.admin.listUsers({ page, perPage: 100 });
+ if (error) {
+ throw new Error(`Failed to resolve local owner from email: ${error.message}`);
+ }
+ if (!data?.users?.length) break;
+
+ const found = data.users.find((user) => user.email?.toLowerCase() === normalizedEmail);
+ if (found?.id) return found.id;
+
+ page = typeof data.nextPage === "number" ? data.nextPage : 0;
+ }
+
+ return null;
+}
+
+async function resolveOwnerFromDocuments(supabase: AdminClient) {
+ const { data, error } = await supabase
+ .from("documents")
+ .select("owner_id")
+ .not("owner_id", "is", null)
+ .order("created_at", { ascending: false })
+ .limit(1)
+ .maybeSingle();
+
+ if (error) {
+ throw new Error(`Failed to resolve local owner fallback from documents: ${error.message}`);
+ }
+
+ return data?.owner_id ?? null;
+}
diff --git a/src/lib/supabase/client.tsx b/src/lib/supabase/client.tsx
index 8a9d5fae2..28cc1ac82 100644
--- a/src/lib/supabase/client.tsx
+++ b/src/lib/supabase/client.tsx
@@ -18,6 +18,8 @@ type AuthContextValue = {
markSessionExpired: () => void;
};
+export const AUTH_EMAIL_STORAGE_KEY = "clinical.dashboard.lastAuthEmail";
+
const AuthContext = createContext(null);
let browserSupabaseClient: SupabaseClient | null | undefined;
let browserSupabaseClientConfig: string | null = null;
@@ -62,6 +64,21 @@ export function authorizationHeadersForAccessToken(accessToken: string | null |
return headers;
}
+function clearLocationHash() {
+ if (typeof window === "undefined") return;
+ if (!window.location.hash) return;
+ window.history.replaceState({}, "", `${window.location.pathname}${window.location.search}`);
+}
+
+function isExpiredOtpError(errorCode: string | null, message: string) {
+ const normalizedMessage = message.toLowerCase();
+ return (
+ errorCode === "otp_expired" ||
+ normalizedMessage.includes("expired") ||
+ normalizedMessage.includes("invalid or has expired")
+ );
+}
+
export function AuthProvider({ children }: { children: ReactNode }) {
const client = useMemo(() => createBrowserSupabaseClient(), []);
const [session, setSession] = useState(null);
@@ -72,9 +89,83 @@ export function AuthProvider({ children }: { children: ReactNode }) {
if (!client) return () => undefined;
let active = true;
- client.auth
- .getSession()
- .then(({ data, error: sessionError }) => {
+
+ const initializeSession = async () => {
+ if (typeof window !== "undefined") {
+ const hash = window.location.hash.startsWith("#") ? window.location.hash.slice(1) : window.location.hash;
+ const callbackParams = new URLSearchParams(hash);
+ const hasCallbackParams =
+ callbackParams.size > 0 &&
+ (callbackParams.has("access_token") ||
+ callbackParams.has("refresh_token") ||
+ callbackParams.has("type") ||
+ callbackParams.has("error") ||
+ callbackParams.has("error_code") ||
+ callbackParams.has("code"));
+
+ if (hasCallbackParams) {
+ const hasCallbackError = callbackParams.has("error") || callbackParams.has("error_code");
+ if (hasCallbackError) {
+ const errorCode = callbackParams.get("error_code");
+ const rawDescription = callbackParams.get("error_description");
+ const message = rawDescription
+ ? decodeURIComponent(rawDescription.replace(/\+/g, " "))
+ : "Sign-in verification failed.";
+ const expired = isExpiredOtpError(errorCode, message);
+ setSession(null);
+ setStatus(expired ? "expired" : "error");
+ setError(expired ? "This sign-in link is invalid or has expired. Send a new one." : message);
+ clearLocationHash();
+ return;
+ }
+
+ type AuthCallbackResult = {
+ data?: {
+ session?: Session | null;
+ };
+ error?: { message?: string } | null;
+ };
+
+ const getSessionFromUrl = (
+ client.auth as {
+ getSessionFromUrl?: () => Promise;
+ }
+ ).getSessionFromUrl;
+ const callbackResult = getSessionFromUrl
+ ? await getSessionFromUrl()
+ : await client.auth.setSession({
+ access_token: decodeURIComponent(callbackParams.get("access_token") ?? ""),
+ refresh_token: decodeURIComponent(callbackParams.get("refresh_token") ?? ""),
+ });
+ if (!active) return;
+ clearLocationHash();
+
+ if (!callbackResult || callbackResult.error) {
+ const message = callbackResult?.error?.message ?? "Sign-in verification failed.";
+ const expired = isExpiredOtpError(callbackParams.get("error_code"), message);
+ setSession(null);
+ setStatus(expired ? "expired" : "error");
+ setError(expired ? "This sign-in link is invalid or has expired. Send a new one." : message);
+ return;
+ }
+
+ const callbackSession = callbackResult?.data?.session;
+ if (callbackSession) {
+ setSession(callbackSession);
+ setStatus("authenticated");
+ setError(null);
+ return;
+ }
+
+ setSession(null);
+ setStatus("signed_out");
+ setError("Sign-in verification did not return a session.");
+ return;
+ }
+ }
+
+ try {
+ const { data, error: sessionError } = await client.auth.getSession();
if (!active) return;
if (sessionError) {
setStatus("error");
@@ -84,12 +175,14 @@ export function AuthProvider({ children }: { children: ReactNode }) {
setSession(data.session);
setStatus(data.session ? "authenticated" : "signed_out");
setError(null);
- })
- .catch(() => {
+ } catch {
if (!active) return;
setStatus("error");
setError("Session could not be loaded.");
- });
+ }
+ };
+
+ void initializeSession();
const {
data: { subscription },
@@ -113,6 +206,12 @@ export function AuthProvider({ children }: { children: ReactNode }) {
return;
}
+ try {
+ window.localStorage.setItem(AUTH_EMAIL_STORAGE_KEY, email);
+ } catch {
+ // localStorage may be unavailable in restrictive browser modes.
+ }
+
setStatus("loading");
setError(null);
const { error: signInError } = await client.auth.signInWithOtp({
diff --git a/src/lib/supabase/project.ts b/src/lib/supabase/project.ts
index bb6c7d39f..65be9f9ac 100644
--- a/src/lib/supabase/project.ts
+++ b/src/lib/supabase/project.ts
@@ -88,9 +88,7 @@ export function checkSupabaseProjectConfig(
}
if (!urlRef) {
- problems.push(
- `NEXT_PUBLIC_SUPABASE_URL must be a Supabase project URL for ${expectedSupabaseProject.name}.`,
- );
+ problems.push(`NEXT_PUBLIC_SUPABASE_URL must be a Supabase project URL for ${expectedSupabaseProject.name}.`);
} else if (urlRef !== expectedSupabaseProject.ref) {
problems.push(
`NEXT_PUBLIC_SUPABASE_URL points to Supabase ref ${urlRef}; expected ${expectedSupabaseProject.ref} (${expectedSupabaseProject.name}).`,
diff --git a/src/lib/types.ts b/src/lib/types.ts
index a3b96daa7..f993daa7a 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -10,10 +10,18 @@ export type ImageEvidenceCategory =
| "medication_chart"
| "graph"
| "screenshot_ui"
+ | "cover_page"
| "photo"
| "logo_decorative"
| "unclear";
+export type ClinicalImageUseClass =
+ | "clinical_evidence"
+ | "administrative"
+ | "reference"
+ | "decorative_or_empty"
+ | "ambiguous";
+
export type DocumentLabelType =
| "topic"
| "document_type"
@@ -111,7 +119,18 @@ export type ChunkImage = {
searchable?: boolean;
clinical_relevance_score?: number;
source_kind?: string | null;
+ sourceKind?: string | null;
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableTextSnippet?: string | null;
+ clinicalUseClass?: ClinicalImageUseClass | null;
+ clinicalUseReason?: string | null;
+ accessibleTableMarkdown?: string | null;
+ tableRows?: string[][] | null;
+ tableColumns?: string[] | null;
labels?: string[];
+ metadata?: Record | null;
};
export type SearchResult = {
@@ -127,11 +146,34 @@ export type SearchResult = {
similarity: number;
text_rank?: number;
hybrid_score?: number;
+ rrf_score?: number;
+ score_explanation?: SearchScoreExplanation;
source_strength?: SourceStrength;
source_metadata?: ClinicalSourceMetadata | null;
+ document_labels?: DocumentLabel[];
+ document_summary?: string | null;
+ adjacent_context?: string | null;
+ memory_cards?: DocumentMemoryCard[];
+ memory_score?: number;
+ relevance?: SourceEvidenceRelevance;
images: ChunkImage[];
};
+export type SearchScoreExplanation = {
+ vectorScore: number;
+ textRank: number;
+ weightedHybridScore: number;
+ rrfScore: number | null;
+ memoryBoost: number;
+ titleBoost: number;
+ metadataBoost: number;
+ clinicalSignalBoost: number;
+ penalty: number;
+ finalScore: number;
+ finalRank?: number;
+ strategy: "weighted_hybrid_served_rrf_telemetry";
+};
+
export type Citation = {
chunk_id: string;
document_id: string;
@@ -151,6 +193,29 @@ export type QuoteCard = Citation & {
export type SourceStrength = "strong" | "moderate" | "limited";
+export type EvidenceRelevanceVerdict = "direct" | "partial" | "nearby" | "none";
+
+export type EvidenceRelevance = {
+ verdict: EvidenceRelevanceVerdict;
+ label: string;
+ matchedTerms: string[];
+ missingTerms: string[];
+ directSourceCount: number;
+ weakSourceCount: number;
+ score: number;
+ supportReason: string;
+ isSourceBacked: boolean;
+};
+
+export type SourceEvidenceRelevance = EvidenceRelevance & {
+ coverageScore: number;
+ rankScore: number;
+ titleMatchedTerms: string[];
+ contentMatchedTerms: string[];
+ metadataMatchedTerms: string[];
+ chips: string[];
+};
+
export type VisualEvidenceCard = {
id: string;
image_id: string;
@@ -165,7 +230,18 @@ export type VisualEvidenceCard = {
viewer_href: string;
image_type?: ImageEvidenceCategory;
clinical_relevance_score?: number;
+ source_kind?: string | null;
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableTextSnippet?: string | null;
+ clinicalUseClass?: ClinicalImageUseClass | null;
+ clinicalUseReason?: string | null;
+ accessibleTableMarkdown?: string | null;
+ tableRows?: string[][] | null;
+ tableColumns?: string[] | null;
labels?: string[];
+ relevance?: SourceEvidenceRelevance;
};
export type DocumentLabel = {
@@ -204,6 +280,73 @@ export type DocumentSummary = {
updated_at?: string;
};
+export type RagIndexingVersion = "rag-universal-v1" | "rag-deep-memory-v1";
+
+export type DocumentIndexQuality = {
+ indexingVersion?: RagIndexingVersion | string | null;
+ memoryVersion?: RagIndexingVersion | string | null;
+ extractionQuality?: ClinicalSourceMetadata["extraction_quality"];
+ sectionCount?: number;
+ memoryCardCount?: number;
+ missingEmbeddings?: number;
+ stale?: boolean;
+};
+
+export type DocumentSectionMemory = {
+ id?: string;
+ document_id: string;
+ owner_id?: string | null;
+ section_index: number;
+ heading: string;
+ heading_path: string[];
+ page_start: number | null;
+ page_end: number | null;
+ chunk_ids: string[];
+ summary: string;
+ tags: string[];
+ extraction_quality: ClinicalSourceMetadata["extraction_quality"];
+ metadata?: Record;
+ created_at?: string;
+ updated_at?: string;
+};
+
+export type DocumentMemoryCardType =
+ | "section_summary"
+ | "table_row"
+ | "threshold"
+ | "medication"
+ | "risk"
+ | "workflow"
+ | "definition"
+ | "citation_anchor";
+
+export type DocumentMemoryCard = {
+ id?: string;
+ document_id: string;
+ owner_id?: string | null;
+ section_id?: string | null;
+ card_type: DocumentMemoryCardType;
+ title: string;
+ content: string;
+ normalized_terms: string[];
+ page_number: number | null;
+ source_chunk_ids: string[];
+ source_image_ids: string[];
+ confidence: number;
+ metadata?: Record;
+ embedding?: number[];
+ created_at?: string;
+ updated_at?: string;
+};
+
+export type RagQueryClass =
+ | "document_lookup"
+ | "table_threshold"
+ | "medication_dose_risk"
+ | "comparison"
+ | "broad_summary"
+ | "unsupported_or_general";
+
export type RelatedDocument = {
document_id: string;
title: string;
@@ -213,10 +356,26 @@ export type RelatedDocument = {
best_pages: number[];
best_chunk_ids: string[];
image_count: number;
+ table_count?: number;
match_reason: string;
score: number;
};
+export type DocumentMatch = {
+ document_id: string;
+ title: string;
+ file_name: string;
+ labels: DocumentLabel[];
+ summarySnippet: string | null;
+ bestPages: number[];
+ bestChunkIds: string[];
+ imageCount: number;
+ tableCount: number;
+ matchReason: string;
+ score: number;
+ relevance?: SourceEvidenceRelevance;
+};
+
export type DocumentBreakdown = {
document_id: string;
title: string;
@@ -237,6 +396,7 @@ export type BestSourceRecommendation = Citation & {
section_heading: string | null;
image_count: number;
viewer_href: string;
+ relevance?: SourceEvidenceRelevance;
};
export type EvidenceSummary = {
@@ -287,6 +447,52 @@ export type SmartPanel = {
sourceCoverage: SourceCoverage;
conflictsOrGaps: ConflictOrGap[];
relatedDocuments?: RelatedDocument[];
+ relevance?: EvidenceRelevance;
+};
+
+export type SmartRagSourceLink = {
+ id: string;
+ label: string;
+ href: string;
+ document_id: string;
+ chunk_id: string;
+ title: string;
+ file_name: string;
+ page_number: number | null;
+ source_strength: SourceStrength;
+ reason: string;
+ snippet: string;
+};
+
+export type SmartRagApiPlan = {
+ query: string;
+ queryClass: RagQueryClass;
+ intent:
+ | "find_document"
+ | "find_threshold_or_table"
+ | "medication_or_risk_answer"
+ | "compare_sources"
+ | "summarize_sources"
+ | "general_or_unsupported";
+ responseMode:
+ | "document_lookup"
+ | "extractive_answer"
+ | "fast_grounded_answer"
+ | "strong_synthesis"
+ | "multi_document_synthesis"
+ | "unsupported";
+ retrievalStrategy:
+ | "search_cache"
+ | "text_fast_path"
+ | "document_lookup_fast_path"
+ | "hybrid"
+ | "vector_fallback"
+ | "unknown";
+ latencyPlan: "cache_or_text_first" | "balanced_hybrid" | "strong_generation" | "no_supported_answer";
+ answerFocus: string;
+ sourceLinkCount: number;
+ coreSourceLinks: SmartRagSourceLink[];
+ streamPlan: string[];
};
export type RagAnswer = {
@@ -298,6 +504,7 @@ export type RagAnswer = {
modelUsed?: string | null;
routingMode?: "unsupported" | "extractive" | "fast" | "strong";
routingReason?: string;
+ queryClass?: RagQueryClass;
latencyTimings?: {
search_cache_hit?: boolean;
text_fast_path_latency_ms?: number;
@@ -306,6 +513,7 @@ export type RagAnswer = {
embedding_cache_hit?: boolean;
supabase_rpc_latency_ms?: number;
rerank_latency_ms?: number;
+ context_pack_latency_ms?: number;
search_latency_ms?: number;
generation_latency_ms?: number;
total_latency_ms?: number;
@@ -322,6 +530,17 @@ export type RagAnswer = {
documentBreakdown?: DocumentBreakdown[];
smartPanel?: SmartPanel;
relatedDocuments?: RelatedDocument[];
+ relevance?: EvidenceRelevance;
+ memoryCardsUsed?: DocumentMemoryCard[];
+ indexingVersion?: RagIndexingVersion | string | null;
+ indexingQuality?: DocumentIndexQuality;
+ smartApiPlan?: SmartRagApiPlan;
+ scoreExplanations?: Array<{
+ chunk_id: string;
+ document_id: string;
+ finalScore: number;
+ score_explanation?: SearchScoreExplanation;
+ }>;
};
export type ExtractedPage = {
@@ -337,7 +556,7 @@ export type ExtractedImage = {
bbox?: [number, number, number, number] | null;
width?: number | null;
height?: number | null;
- sourceKind?: "embedded" | "diagram_crop" | "page_region" | "fallback";
+ sourceKind?: "embedded" | "table_crop" | "diagram_crop" | "page_region" | "fallback" | "cover_page";
metadata?: Record;
};
@@ -351,7 +570,18 @@ export type ChunkInput = {
documentId: string;
pageNumber: number | null;
pageText: string;
- images?: Array<{ id: string; caption: string; pageNumber: number | null }>;
+ images?: Array<{
+ id: string;
+ caption: string;
+ pageNumber: number | null;
+ imageType?: ImageEvidenceCategory;
+ sourceKind?: string | null;
+ labels?: string[];
+ tableLabel?: string | null;
+ tableTitle?: string | null;
+ tableRole?: string | null;
+ tableTextSnippet?: string | null;
+ }>;
metadata?: Record;
};
diff --git a/src/lib/ward-output.ts b/src/lib/ward-output.ts
index 248431252..355326f9f 100644
--- a/src/lib/ward-output.ts
+++ b/src/lib/ward-output.ts
@@ -1,17 +1,33 @@
import { formatCitationLabel } from "@/lib/citations";
import { clipboardProvenanceLine } from "@/lib/source-metadata";
-import type { QuoteCard, RagAnswer } from "@/lib/types";
+import type { QuoteCard, RagAnswer, VisualEvidenceCard } from "@/lib/types";
+
+export type ClinicalOutputSectionId = "action" | "monitoring" | "thresholds" | "escalation" | "verify-source";
+
+export type ClinicalThresholdTable = {
+ id: string;
+ caption: string;
+ markdown?: string | null;
+ rows?: string[][] | null;
+ columns?: string[] | null;
+ sourceLabel?: string;
+};
export type ClinicalOutputSection = {
- id: "key-actions" | "monitoring-checklist" | "escalation-triggers";
+ id: ClinicalOutputSectionId;
title: string;
items: string[];
+ tables?: ClinicalThresholdTable[];
};
+const actionPattern =
+ /\b(action|start|stop|hold|withhold|cease|avoid|review|assess|check|refer|discuss|required|must|should)\b/i;
const escalationPattern =
/\b(escalat|urgent|senior|review|risk|intent|attempt|agitation|supervision|toxicity|vomiting|diarrhoea|dehydration|tremor|confusion|ataxia|fever|chest pain|dyspnoea|seizure|constipation)\b/i;
const monitoringPattern =
/\b(monitor|check|baseline|fbc|anc|renal|thyroid|calcium|metabolic|myocarditis|blood pressure|weight|level|schedule)\b/i;
+const thresholdPattern =
+ /\b(threshold|cut[\s-]?off|withhold|cease|stop|hold|discontinue|anc|fbc|wbc|neutrophil|level|range|criteria|score|rating|below|above|less than|greater than|mmol|mg\/l|x\s*10)\b|[<>]=?|[≤≥]/i;
function normalizeText(text: string) {
return text.replace(/\s+/g, " ").trim();
@@ -32,31 +48,153 @@ function uniqueShortItems(items: string[], limit: number) {
return output;
}
+function parseMarkdownTable(markdown?: string | null) {
+ if (!markdown) return null;
+ const rows = markdown
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line.includes("|") && !/^\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?$/.test(line))
+ .map((line) =>
+ line
+ .replace(/^\|/, "")
+ .replace(/\|$/, "")
+ .split("|")
+ .map((cell) => cell.replace(/\\\|/g, "|").trim()),
+ )
+ .filter((row) => row.some(Boolean));
+ return rows.length ? rows : null;
+}
+
+function tableShape(table: Pick) {
+ const rows = table.rows?.length ? table.rows : parseMarkdownTable(table.markdown);
+ const rowCount = rows?.length ?? 0;
+ const columnCount = Math.max(table.columns?.length ?? 0, ...(rows?.map((row) => row.length) ?? [0]));
+ return { rowCount, columnCount };
+}
+
+function tableHasUsableShape(table: Pick) {
+ const { rowCount, columnCount } = tableShape(table);
+ return rowCount >= 2 && columnCount >= 2;
+}
+
+function visualEvidenceText(card: VisualEvidenceCard) {
+ return [card.tableLabel, card.tableTitle, card.caption, card.tableTextSnippet, card.labels?.join(" ")]
+ .filter(Boolean)
+ .join(" ");
+}
+
+function tableFromVisualEvidence(card: VisualEvidenceCard): ClinicalThresholdTable | null {
+ const markdown = card.accessibleTableMarkdown?.trim()
+ ? card.accessibleTableMarkdown
+ : card.tableTextSnippet?.includes("|")
+ ? card.tableTextSnippet
+ : null;
+ const candidate: ClinicalThresholdTable = {
+ id: card.id,
+ caption: [card.tableLabel, card.tableTitle].filter(Boolean).join(": ") || card.caption,
+ markdown,
+ rows: card.tableRows,
+ columns: card.tableColumns,
+ sourceLabel: `${card.title}, page ${card.page_number ?? "n/a"}`,
+ };
+
+ if (!tableHasUsableShape(candidate)) return null;
+ return candidate;
+}
+
+function buildThresholdTables(answer: RagAnswer) {
+ const evidence = [...(answer.visualEvidence ?? []), ...(answer.smartPanel?.visualEvidence ?? [])];
+ const seen = new Set();
+ const tables: ClinicalThresholdTable[] = [];
+
+ for (const card of evidence) {
+ if (seen.has(card.id)) continue;
+ seen.add(card.id);
+ if (!thresholdPattern.test(visualEvidenceText(card))) continue;
+
+ const table = tableFromVisualEvidence(card);
+ if (!table) continue;
+ tables.push(table);
+ if (tables.length >= 2) break;
+ }
+
+ return tables;
+}
+
+function buildVerifySourceItems(answer: RagAnswer) {
+ const citationCount = answer.citations.length;
+ const quoteCount = answer.quoteCards?.length ?? 0;
+ const sourceStrength = answer.evidenceSummary?.source_strength;
+ return uniqueShortItems(
+ [
+ citationCount
+ ? `${citationCount} linked citation${citationCount === 1 ? "" : "s"} available for source verification.`
+ : "No linked citations were returned for this answer.",
+ quoteCount
+ ? `${quoteCount} exact source quote${quoteCount === 1 ? "" : "s"} available before clinical use.`
+ : "No exact source quote was returned; verify the answer against the source document.",
+ sourceStrength && sourceStrength !== "none" ? `Strongest retrieved source support: ${sourceStrength}.` : "",
+ answer.confidence === "unsupported"
+ ? "Treat as unsupported: do not use clinically without opening and checking source text."
+ : "Open the cited source passage before copying into the medical record.",
+ ],
+ 4,
+ );
+}
+
export function buildClinicalOutputSections(answer: RagAnswer | null | undefined) {
if (!answer) return [];
const sectionBodies = answer.answerSections?.map((section) => section.body) ?? [];
const quoteTexts = answer.quoteCards?.map((quote) => quote.quote) ?? [];
- const combined = [...sectionBodies, ...quoteTexts];
+ const combined = [...sectionBodies, ...quoteTexts, answer.answer];
- const keyActions = uniqueShortItems(sectionBodies.length ? sectionBodies : [answer.answer], 3);
+ const actionSource = sectionBodies.length
+ ? sectionBodies.filter((item) => actionPattern.test(item) || !monitoringPattern.test(item))
+ : [answer.answer];
+ const actions = uniqueShortItems(actionSource.length ? actionSource : [answer.answer], 3);
const monitoring = uniqueShortItems(
combined.filter((item) => monitoringPattern.test(item)),
4,
);
+ const thresholdItems = uniqueShortItems(
+ combined.filter((item) => thresholdPattern.test(item)),
+ 4,
+ );
+ const thresholdTables = buildThresholdTables(answer);
const escalation = uniqueShortItems(
combined.filter((item) => escalationPattern.test(item)),
4,
);
+ const verifySource = buildVerifySourceItems(answer);
const sections: ClinicalOutputSection[] = [];
- if (keyActions.length) sections.push({ id: "key-actions", title: "Key actions", items: keyActions });
- if (monitoring.length) {
- sections.push({ id: "monitoring-checklist", title: "Monitoring checklist", items: monitoring });
- }
- if (escalation.length) {
- sections.push({ id: "escalation-triggers", title: "Escalation triggers", items: escalation });
- }
+ sections.push({
+ id: "action",
+ title: "Action",
+ items: actions.length ? actions : ["Use the source-backed answer as the starting action and verify citations first."],
+ });
+ sections.push({
+ id: "monitoring",
+ title: "Monitoring",
+ items: monitoring.length ? monitoring : ["No explicit monitoring schedule was extracted from the answer or quotes."],
+ });
+ sections.push({
+ id: "thresholds",
+ title: "Thresholds",
+ items: thresholdItems.length ? thresholdItems : ["No explicit numeric threshold or table row was extracted."],
+ tables: thresholdTables,
+ });
+ sections.push({
+ id: "escalation",
+ title: "Escalation",
+ items: escalation.length ? escalation : ["No explicit escalation trigger was extracted from the answer or quotes."],
+ });
+ sections.push({
+ id: "verify-source",
+ title: "Verify source",
+ items: verifySource,
+ });
return sections;
}
@@ -135,6 +273,7 @@ export function createQuoteFollowUp(quote: QuoteCard) {
export function shouldPollForUpdates(
demoMode: boolean,
visibilityState: DocumentVisibilityState | "visible" | "hidden",
+ hasActiveWork = true,
) {
- return !demoMode && visibilityState === "visible";
+ return hasActiveWork && !demoMode && visibilityState === "visible";
}
diff --git a/supabase/migrations/20260528007000_database_hardening_before_import.sql b/supabase/migrations/20260528007000_database_hardening_before_import.sql
new file mode 100644
index 000000000..6c7104d8e
--- /dev/null
+++ b/supabase/migrations/20260528007000_database_hardening_before_import.sql
@@ -0,0 +1,89 @@
+-- Harden Data API exposure before large imports.
+-- The app uses server-owned API routes for writes, so browser roles only need
+-- read access plus manual document-label edits.
+
+alter default privileges for role postgres in schema public
+ revoke all privileges on tables from anon, authenticated;
+alter default privileges for role postgres in schema public
+ revoke usage, select on sequences from anon, authenticated;
+alter default privileges for role postgres in schema public
+ revoke execute on functions from public, anon, authenticated;
+alter default privileges for role postgres in schema public
+ grant select, insert, update, delete on tables to service_role;
+alter default privileges for role postgres in schema public
+ grant usage, select on sequences to service_role;
+alter default privileges for role postgres in schema public
+ grant execute on functions to service_role;
+
+revoke usage on schema public from anon;
+grant usage on schema public to authenticated, service_role;
+
+revoke all privileges on all tables in schema public from anon, authenticated;
+revoke all privileges on all sequences in schema public from anon, authenticated;
+revoke execute on all functions in schema public from public, anon, authenticated;
+
+grant select, insert, update, delete on all tables in schema public to service_role;
+grant usage, select on all sequences in schema public to service_role;
+grant execute on all functions in schema public to service_role;
+
+grant select on table
+ public.import_batches,
+ public.documents,
+ public.document_pages,
+ public.document_images,
+ public.document_labels,
+ public.document_summaries,
+ public.document_chunks,
+ public.ingestion_jobs,
+ public.rag_queries
+to authenticated;
+
+grant insert, update, delete on table public.document_labels to authenticated;
+
+drop policy if exists "documents owner insert" on public.documents;
+drop policy if exists "documents owner update" on public.documents;
+drop policy if exists "documents owner delete" on public.documents;
+drop policy if exists "rag owner insert" on public.rag_queries;
+
+create table if not exists public.storage_cleanup_jobs (
+ id uuid primary key default gen_random_uuid(),
+ owner_id uuid references auth.users(id) on delete set null,
+ document_id uuid,
+ document_title text,
+ document_bucket text not null default 'clinical-documents',
+ document_paths text[] not null default '{}',
+ image_bucket text not null default 'clinical-images',
+ image_paths text[] not null default '{}',
+ status text not null default 'pending'
+ check (status in ('pending', 'completed', 'failed')),
+ attempts integer not null default 0,
+ storage_removed integer not null default 0,
+ last_error text,
+ metadata jsonb not null default '{}'::jsonb,
+ completed_at timestamptz,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now()
+);
+
+create index if not exists storage_cleanup_jobs_owner_status_idx
+ on public.storage_cleanup_jobs(owner_id, status, created_at desc);
+create index if not exists storage_cleanup_jobs_document_idx
+ on public.storage_cleanup_jobs(document_id);
+create index if not exists rag_queries_source_chunk_ids_gin_idx
+ on public.rag_queries using gin(source_chunk_ids);
+
+drop trigger if exists storage_cleanup_jobs_updated_at on public.storage_cleanup_jobs;
+create trigger storage_cleanup_jobs_updated_at
+before update on public.storage_cleanup_jobs
+for each row execute function public.set_updated_at();
+
+alter table public.storage_cleanup_jobs enable row level security;
+
+drop policy if exists "storage cleanup owner read" on public.storage_cleanup_jobs;
+create policy "storage cleanup owner read" on public.storage_cleanup_jobs
+for select to authenticated
+using ((select auth.uid()) = owner_id);
+
+grant select, insert, update, delete on table public.storage_cleanup_jobs to service_role;
+revoke all privileges on table public.storage_cleanup_jobs from anon, authenticated;
+grant select on table public.storage_cleanup_jobs to authenticated;
diff --git a/supabase/migrations/20260528008000_table_visual_metadata.sql b/supabase/migrations/20260528008000_table_visual_metadata.sql
new file mode 100644
index 000000000..fae914914
--- /dev/null
+++ b/supabase/migrations/20260528008000_table_visual_metadata.sql
@@ -0,0 +1,37 @@
+create or replace function public.chunk_image_metadata(chunk_image_ids uuid[])
+returns jsonb
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ select coalesce(
+ jsonb_agg(
+ jsonb_build_object(
+ 'id', i.id,
+ 'page_number', i.page_number,
+ 'storage_path', i.storage_path,
+ 'caption', i.caption,
+ 'bbox', i.bbox,
+ 'image_type', i.image_type,
+ 'searchable', i.searchable,
+ 'clinical_relevance_score', i.clinical_relevance_score,
+ 'source_kind', i.source_kind,
+ 'sourceKind', i.source_kind,
+ 'tableLabel', nullif(i.metadata->>'table_label', ''),
+ 'tableTitle', nullif(i.metadata->>'table_title', ''),
+ 'tableRole', nullif(i.metadata->>'table_role', ''),
+ 'tableTextSnippet', nullif(left(coalesce(i.metadata->>'table_text_snippet', i.metadata->>'table_text', ''), 500), ''),
+ 'width', i.width,
+ 'height', i.height,
+ 'labels', i.labels,
+ 'metadata', i.metadata
+ )
+ order by i.page_number nulls last, i.created_at
+ ),
+ '[]'::jsonb
+ )
+ from public.document_images i
+ where i.id = any(chunk_image_ids)
+ and i.searchable = true
+ and i.image_type <> 'logo_decorative';
+$$;
diff --git a/supabase/migrations/20260528009000_deep_memory_indexing.sql b/supabase/migrations/20260528009000_deep_memory_indexing.sql
new file mode 100644
index 000000000..faaca1d73
--- /dev/null
+++ b/supabase/migrations/20260528009000_deep_memory_indexing.sql
@@ -0,0 +1,147 @@
+create table if not exists public.document_sections (
+ id uuid primary key default gen_random_uuid(),
+ document_id uuid not null references public.documents(id) on delete cascade,
+ owner_id uuid references auth.users(id) on delete set null,
+ section_index integer not null,
+ heading text not null,
+ heading_path text[] not null default '{}',
+ page_start integer,
+ page_end integer,
+ chunk_ids uuid[] not null default '{}',
+ summary text not null default '',
+ tags text[] not null default '{}',
+ extraction_quality text not null default 'unknown'
+ check (extraction_quality in ('good', 'partial', 'poor', 'unknown')),
+ metadata jsonb not null default '{}'::jsonb,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (document_id, section_index)
+);
+
+create table if not exists public.document_memory_cards (
+ id uuid primary key default gen_random_uuid(),
+ document_id uuid not null references public.documents(id) on delete cascade,
+ owner_id uuid references auth.users(id) on delete set null,
+ section_id uuid references public.document_sections(id) on delete set null,
+ card_type text not null
+ check (card_type in (
+ 'section_summary',
+ 'table_row',
+ 'threshold',
+ 'medication',
+ 'risk',
+ 'workflow',
+ 'definition',
+ 'citation_anchor'
+ )),
+ title text not null,
+ content text not null,
+ normalized_terms text[] not null default '{}',
+ page_number integer,
+ source_chunk_ids uuid[] not null default '{}',
+ source_image_ids uuid[] not null default '{}',
+ confidence real not null default 0.5 check (confidence >= 0 and confidence <= 1),
+ metadata jsonb not null default '{}'::jsonb,
+ embedding vector(1536) not null,
+ search_tsv tsvector generated always as (
+ to_tsvector(
+ 'english',
+ coalesce(title, '') || ' ' || coalesce(content, '')
+ )
+ ) stored,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now()
+);
+
+create or replace function public.set_updated_at()
+returns trigger
+language plpgsql
+set search_path = public, extensions, pg_temp
+as $$
+begin
+ new.updated_at = now();
+ return new;
+end;
+$$;
+
+create index if not exists document_sections_document_idx
+ on public.document_sections(document_id, section_index);
+create index if not exists document_sections_chunk_ids_gin_idx
+ on public.document_sections using gin(chunk_ids);
+create index if not exists document_sections_tags_gin_idx
+ on public.document_sections using gin(tags);
+create index if not exists document_memory_cards_document_idx
+ on public.document_memory_cards(document_id, card_type, confidence desc);
+create index if not exists document_memory_cards_search_idx
+ on public.document_memory_cards using gin(search_tsv);
+create index if not exists document_memory_cards_terms_idx
+ on public.document_memory_cards using gin(normalized_terms);
+create index if not exists document_memory_cards_source_chunks_idx
+ on public.document_memory_cards using gin(source_chunk_ids);
+create index if not exists document_memory_cards_source_images_idx
+ on public.document_memory_cards using gin(source_image_ids);
+create index if not exists document_memory_cards_embedding_hnsw_idx
+ on public.document_memory_cards using hnsw (embedding vector_cosine_ops);
+
+drop trigger if exists document_sections_updated_at on public.document_sections;
+create trigger document_sections_updated_at
+before update on public.document_sections
+for each row execute function public.set_updated_at();
+
+drop trigger if exists document_memory_cards_updated_at on public.document_memory_cards;
+create trigger document_memory_cards_updated_at
+before update on public.document_memory_cards
+for each row execute function public.set_updated_at();
+
+create or replace function public.reset_document_index(p_document_id uuid)
+returns void
+language plpgsql
+set search_path = public, extensions, pg_temp
+as $$
+begin
+ delete from public.document_memory_cards where document_id = p_document_id;
+ delete from public.document_sections where document_id = p_document_id;
+ delete from public.document_chunks where document_id = p_document_id;
+ delete from public.document_images where document_id = p_document_id;
+ delete from public.document_pages where document_id = p_document_id;
+end;
+$$;
+
+create or replace function public.stamp_document_deep_memory_version(
+ p_document_id uuid,
+ p_version text
+)
+returns void
+language plpgsql
+set search_path = public, extensions, pg_temp
+as $$
+declare
+ stamped_at timestamptz := now();
+begin
+ update public.documents
+ set metadata = coalesce(metadata, '{}'::jsonb) || jsonb_build_object(
+ 'rag_indexing_version', p_version,
+ 'rag_memory_version', p_version,
+ 'rag_memory_updated_at', stamped_at
+ )
+ where id = p_document_id;
+
+ update public.document_chunks
+ set metadata = coalesce(metadata, '{}'::jsonb) || jsonb_build_object(
+ 'rag_indexing_version', p_version,
+ 'rag_memory_version', p_version,
+ 'rag_memory_updated_at', stamped_at
+ )
+ where document_id = p_document_id;
+end;
+$$;
+
+revoke all privileges on table public.document_sections from anon, authenticated;
+revoke all privileges on table public.document_memory_cards from anon, authenticated;
+grant select, insert, update, delete on table public.document_sections to service_role;
+grant select, insert, update, delete on table public.document_memory_cards to service_role;
+grant execute on function public.stamp_document_deep_memory_version(uuid, text) to service_role;
+grant execute on function public.reset_document_index(uuid) to service_role;
+
+alter table public.document_sections enable row level security;
+alter table public.document_memory_cards enable row level security;
diff --git a/supabase/migrations/20260602001000_rag_search_hardening.sql b/supabase/migrations/20260602001000_rag_search_hardening.sql
new file mode 100644
index 000000000..96e3bcdbc
--- /dev/null
+++ b/supabase/migrations/20260602001000_rag_search_hardening.sql
@@ -0,0 +1,345 @@
+drop function if exists public.match_document_chunks_hybrid(vector, text, integer, double precision, uuid[], uuid);
+
+create or replace function public.match_document_chunks_hybrid(
+ query_embedding vector(1536),
+ query_text text,
+ match_count integer default 12,
+ min_similarity double precision default 0.12,
+ document_filters uuid[] default null,
+ owner_filter uuid default null
+)
+returns table (
+ id uuid,
+ document_id uuid,
+ title text,
+ file_name text,
+ page_number integer,
+ chunk_index integer,
+ section_heading text,
+ content text,
+ image_ids uuid[],
+ source_metadata jsonb,
+ similarity double precision,
+ text_rank double precision,
+ hybrid_score double precision,
+ rrf_score double precision,
+ images jsonb
+)
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ with query as (
+ select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
+ ),
+ vector_ranked as (
+ select
+ c.id,
+ c.document_id,
+ c.page_number,
+ c.chunk_index,
+ c.section_heading,
+ c.content,
+ c.image_ids,
+ 1 - (c.embedding <=> query_embedding) as similarity,
+ (
+ ts_rank_cd(c.search_tsv, query.tsq) +
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
+ )::double precision as text_rank,
+ row_number() over (order by c.embedding <=> query_embedding) as vector_rank,
+ null::bigint as text_match_rank
+ from public.document_chunks c
+ join public.documents d on d.id = c.document_id
+ cross join query
+ where (document_filters is null or c.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and 1 - (c.embedding <=> query_embedding) >= min_similarity
+ order by c.embedding <=> query_embedding
+ limit greatest(match_count * 6, 48)
+ ),
+ text_ranked as (
+ select
+ c.id,
+ c.document_id,
+ c.page_number,
+ c.chunk_index,
+ c.section_heading,
+ c.content,
+ c.image_ids,
+ 1 - (c.embedding <=> query_embedding) as similarity,
+ (
+ ts_rank_cd(c.search_tsv, query.tsq) +
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
+ )::double precision as text_rank,
+ null::bigint as vector_rank,
+ row_number() over (
+ order by
+ (
+ ts_rank_cd(c.search_tsv, query.tsq) +
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
+ ) desc,
+ c.embedding <=> query_embedding
+ ) as text_match_rank
+ from public.document_chunks c
+ join public.documents d on d.id = c.document_id
+ cross join query
+ where (document_filters is null or c.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and (c.search_tsv @@ query.tsq or d.title_search_tsv @@ query.tsq)
+ order by (
+ ts_rank_cd(c.search_tsv, query.tsq) +
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
+ ) desc
+ limit greatest(match_count * 6, 48)
+ ),
+ combined as (
+ select * from vector_ranked
+ union all
+ select * from text_ranked
+ ),
+ scored as (
+ select
+ id,
+ document_id,
+ page_number,
+ chunk_index,
+ section_heading,
+ content,
+ image_ids,
+ max(similarity)::double precision as similarity,
+ max(text_rank)::double precision as text_rank,
+ min(vector_rank) as vector_rank,
+ min(text_match_rank) as text_match_rank
+ from combined
+ group by id, document_id, page_number, chunk_index, section_heading, content, image_ids
+ )
+ select
+ c.id,
+ c.document_id,
+ d.title,
+ d.file_name,
+ c.page_number,
+ c.chunk_index,
+ c.section_heading,
+ c.content,
+ c.image_ids,
+ d.metadata as source_metadata,
+ c.similarity,
+ c.text_rank,
+ ((c.similarity * 0.72) + (least(c.text_rank, 1) * 0.28))::double precision as hybrid_score,
+ (
+ coalesce(1.0 / (60 + c.vector_rank), 0) +
+ coalesce(1.0 / (60 + c.text_match_rank), 0)
+ )::double precision as rrf_score,
+ public.chunk_image_metadata(c.image_ids) as images
+ from scored c
+ join public.documents d on d.id = c.document_id
+ order by hybrid_score desc, c.similarity desc, c.text_rank desc
+ limit match_count;
+$$;
+
+create or replace function public.match_document_memory_cards_hybrid(
+ query_embedding vector(1536),
+ query_text text,
+ match_count integer default 32,
+ min_similarity double precision default 0.1,
+ document_filters uuid[] default null,
+ owner_filter uuid default null
+)
+returns table (
+ id uuid,
+ document_id uuid,
+ owner_id uuid,
+ section_id uuid,
+ card_type text,
+ title text,
+ content text,
+ normalized_terms text[],
+ page_number integer,
+ source_chunk_ids uuid[],
+ source_image_ids uuid[],
+ confidence real,
+ metadata jsonb,
+ similarity double precision,
+ text_rank double precision,
+ hybrid_score double precision,
+ rrf_score double precision
+)
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ with query as (
+ select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
+ ),
+ vector_ranked as (
+ select
+ m.*,
+ 1 - (m.embedding <=> query_embedding) as similarity,
+ ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank,
+ row_number() over (order by m.embedding <=> query_embedding) as vector_rank,
+ null::bigint as text_match_rank
+ from public.document_memory_cards m
+ join public.documents d on d.id = m.document_id
+ cross join query
+ where (document_filters is null or m.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and 1 - (m.embedding <=> query_embedding) >= min_similarity
+ order by m.embedding <=> query_embedding
+ limit greatest(match_count * 4, 64)
+ ),
+ text_ranked as (
+ select
+ m.*,
+ 1 - (m.embedding <=> query_embedding) as similarity,
+ ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank,
+ null::bigint as vector_rank,
+ row_number() over (
+ order by ts_rank_cd(m.search_tsv, query.tsq) desc, m.embedding <=> query_embedding
+ ) as text_match_rank
+ from public.document_memory_cards m
+ join public.documents d on d.id = m.document_id
+ cross join query
+ where (document_filters is null or m.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and m.search_tsv @@ query.tsq
+ order by ts_rank_cd(m.search_tsv, query.tsq) desc
+ limit greatest(match_count * 4, 64)
+ ),
+ combined as (
+ select * from vector_ranked
+ union all
+ select * from text_ranked
+ ),
+ scored as (
+ select
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata,
+ max(similarity)::double precision as similarity,
+ max(text_rank)::double precision as text_rank,
+ min(vector_rank) as vector_rank,
+ min(text_match_rank) as text_match_rank
+ from combined
+ group by
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata
+ )
+ select
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata,
+ similarity,
+ text_rank,
+ ((similarity * 0.65) + (least(text_rank, 1) * 0.25) + (confidence * 0.10))::double precision as hybrid_score,
+ (
+ coalesce(1.0 / (60 + vector_rank), 0) +
+ coalesce(1.0 / (60 + text_match_rank), 0)
+ )::double precision as rrf_score
+ from scored
+ order by hybrid_score desc, similarity desc, text_rank desc, confidence desc
+ limit match_count;
+$$;
+
+create or replace function public.match_documents_for_query(
+ query_text text,
+ match_count integer default 12,
+ owner_filter uuid default null
+)
+returns table (
+ id uuid,
+ owner_id uuid,
+ title text,
+ file_name text,
+ status text,
+ page_count integer,
+ chunk_count integer,
+ image_count integer,
+ metadata jsonb,
+ text_rank double precision,
+ match_reason text
+)
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ with query as (
+ select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
+ ),
+ ranked as (
+ select
+ d.id,
+ d.owner_id,
+ d.title,
+ d.file_name,
+ d.status,
+ d.page_count,
+ d.chunk_count,
+ d.image_count,
+ d.metadata,
+ (
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 4.0) +
+ (ts_rank_cd(d.search_tsv, query.tsq) * 1.5)
+ )::double precision as text_rank,
+ case
+ when d.title_search_tsv @@ query.tsq then 'title'
+ when d.search_tsv @@ query.tsq then 'metadata'
+ else 'none'
+ end as match_reason
+ from public.documents d
+ cross join query
+ where (owner_filter is null or d.owner_id = owner_filter)
+ and d.status = 'indexed'
+ and (d.title_search_tsv @@ query.tsq or d.search_tsv @@ query.tsq)
+ )
+ select *
+ from ranked
+ where text_rank > 0
+ order by text_rank desc, page_count desc, title asc
+ limit match_count;
+$$;
+
+revoke execute on function public.match_document_chunks_hybrid(vector, text, integer, double precision, uuid[], uuid)
+ from anon, authenticated, public;
+revoke execute on function public.match_document_memory_cards_hybrid(vector, text, integer, double precision, uuid[], uuid)
+ from anon, authenticated, public;
+revoke execute on function public.match_documents_for_query(text, integer, uuid)
+ from anon, authenticated, public;
+
+grant execute on function public.match_document_chunks_hybrid(vector, text, integer, double precision, uuid[], uuid)
+ to service_role;
+grant execute on function public.match_document_memory_cards_hybrid(vector, text, integer, double precision, uuid[], uuid)
+ to service_role;
+grant execute on function public.match_documents_for_query(text, integer, uuid)
+ to service_role;
diff --git a/supabase/schema.sql b/supabase/schema.sql
index 6d23d9aaf..671648e46 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -190,6 +190,61 @@ create table if not exists public.document_summaries (
updated_at timestamptz not null default now()
);
+create table if not exists public.document_sections (
+ id uuid primary key default gen_random_uuid(),
+ document_id uuid not null references public.documents(id) on delete cascade,
+ owner_id uuid references auth.users(id) on delete set null,
+ section_index integer not null,
+ heading text not null,
+ heading_path text[] not null default '{}',
+ page_start integer,
+ page_end integer,
+ chunk_ids uuid[] not null default '{}',
+ summary text not null default '',
+ tags text[] not null default '{}',
+ extraction_quality text not null default 'unknown'
+ check (extraction_quality in ('good', 'partial', 'poor', 'unknown')),
+ metadata jsonb not null default '{}'::jsonb,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (document_id, section_index)
+);
+
+create table if not exists public.document_memory_cards (
+ id uuid primary key default gen_random_uuid(),
+ document_id uuid not null references public.documents(id) on delete cascade,
+ owner_id uuid references auth.users(id) on delete set null,
+ section_id uuid references public.document_sections(id) on delete set null,
+ card_type text not null
+ check (card_type in (
+ 'section_summary',
+ 'table_row',
+ 'threshold',
+ 'medication',
+ 'risk',
+ 'workflow',
+ 'definition',
+ 'citation_anchor'
+ )),
+ title text not null,
+ content text not null,
+ normalized_terms text[] not null default '{}',
+ page_number integer,
+ source_chunk_ids uuid[] not null default '{}',
+ source_image_ids uuid[] not null default '{}',
+ confidence real not null default 0.5 check (confidence >= 0 and confidence <= 1),
+ metadata jsonb not null default '{}'::jsonb,
+ embedding vector(1536) not null,
+ search_tsv tsvector generated always as (
+ to_tsvector(
+ 'english',
+ coalesce(title, '') || ' ' || coalesce(content, '')
+ )
+ ) stored,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now()
+);
+
create table if not exists public.document_chunks (
id uuid primary key default gen_random_uuid(),
document_id uuid not null references public.documents(id) on delete cascade,
@@ -239,6 +294,26 @@ create table if not exists public.rag_queries (
created_at timestamptz not null default now()
);
+create table if not exists public.storage_cleanup_jobs (
+ id uuid primary key default gen_random_uuid(),
+ owner_id uuid references auth.users(id) on delete set null,
+ document_id uuid,
+ document_title text,
+ document_bucket text not null default 'clinical-documents',
+ document_paths text[] not null default '{}',
+ image_bucket text not null default 'clinical-images',
+ image_paths text[] not null default '{}',
+ status text not null default 'pending'
+ check (status in ('pending', 'completed', 'failed')),
+ attempts integer not null default 0,
+ storage_removed integer not null default 0,
+ last_error text,
+ metadata jsonb not null default '{}'::jsonb,
+ completed_at timestamptz,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now()
+);
+
create unique index if not exists documents_owner_content_hash_unique_idx
on public.documents(owner_id, content_hash)
where content_hash is not null;
@@ -264,6 +339,24 @@ create index if not exists document_labels_owner_label_idx
on public.document_labels(owner_id, label_type, label);
create index if not exists document_labels_document_idx on public.document_labels(document_id);
create index if not exists document_summaries_owner_idx on public.document_summaries(owner_id, generated_at desc);
+create index if not exists document_sections_document_idx
+ on public.document_sections(document_id, section_index);
+create index if not exists document_sections_chunk_ids_gin_idx
+ on public.document_sections using gin(chunk_ids);
+create index if not exists document_sections_tags_gin_idx
+ on public.document_sections using gin(tags);
+create index if not exists document_memory_cards_document_idx
+ on public.document_memory_cards(document_id, card_type, confidence desc);
+create index if not exists document_memory_cards_search_idx
+ on public.document_memory_cards using gin(search_tsv);
+create index if not exists document_memory_cards_terms_idx
+ on public.document_memory_cards using gin(normalized_terms);
+create index if not exists document_memory_cards_source_chunks_idx
+ on public.document_memory_cards using gin(source_chunk_ids);
+create index if not exists document_memory_cards_source_images_idx
+ on public.document_memory_cards using gin(source_image_ids);
+create index if not exists document_memory_cards_embedding_hnsw_idx
+ on public.document_memory_cards using hnsw (embedding vector_cosine_ops);
create index if not exists document_chunks_document_idx on public.document_chunks(document_id, chunk_index);
create index if not exists document_chunks_search_idx on public.document_chunks using gin(search_tsv);
create index if not exists document_chunks_embedding_hnsw_idx
@@ -274,6 +367,12 @@ create index if not exists ingestion_jobs_claim_idx
on public.ingestion_jobs(status, next_run_at, created_at)
where status in ('pending', 'processing');
create index if not exists rag_queries_owner_idx on public.rag_queries(owner_id, created_at desc);
+create index if not exists rag_queries_source_chunk_ids_gin_idx
+ on public.rag_queries using gin(source_chunk_ids);
+create index if not exists storage_cleanup_jobs_owner_status_idx
+ on public.storage_cleanup_jobs(owner_id, status, created_at desc);
+create index if not exists storage_cleanup_jobs_document_idx
+ on public.storage_cleanup_jobs(document_id);
create or replace function public.set_updated_at()
returns trigger
@@ -306,6 +405,21 @@ create trigger image_caption_cache_updated_at
before update on public.image_caption_cache
for each row execute function public.set_updated_at();
+drop trigger if exists document_sections_updated_at on public.document_sections;
+create trigger document_sections_updated_at
+before update on public.document_sections
+for each row execute function public.set_updated_at();
+
+drop trigger if exists document_memory_cards_updated_at on public.document_memory_cards;
+create trigger document_memory_cards_updated_at
+before update on public.document_memory_cards
+for each row execute function public.set_updated_at();
+
+drop trigger if exists storage_cleanup_jobs_updated_at on public.storage_cleanup_jobs;
+create trigger storage_cleanup_jobs_updated_at
+before update on public.storage_cleanup_jobs
+for each row execute function public.set_updated_at();
+
create or replace function public.claim_ingestion_jobs(
p_worker_id text,
p_claim_limit integer default 1,
@@ -385,12 +499,43 @@ language plpgsql
set search_path = public, extensions, pg_temp
as $$
begin
+ delete from public.document_memory_cards where document_id = p_document_id;
+ delete from public.document_sections where document_id = p_document_id;
delete from public.document_chunks where document_id = p_document_id;
delete from public.document_images where document_id = p_document_id;
delete from public.document_pages where document_id = p_document_id;
end;
$$;
+create or replace function public.stamp_document_deep_memory_version(
+ p_document_id uuid,
+ p_version text
+)
+returns void
+language plpgsql
+set search_path = public, extensions, pg_temp
+as $$
+declare
+ stamped_at timestamptz := now();
+begin
+ update public.documents
+ set metadata = coalesce(metadata, '{}'::jsonb) || jsonb_build_object(
+ 'rag_indexing_version', p_version,
+ 'rag_memory_version', p_version,
+ 'rag_memory_updated_at', stamped_at
+ )
+ where id = p_document_id;
+
+ update public.document_chunks
+ set metadata = coalesce(metadata, '{}'::jsonb) || jsonb_build_object(
+ 'rag_indexing_version', p_version,
+ 'rag_memory_version', p_version,
+ 'rag_memory_updated_at', stamped_at
+ )
+ where document_id = p_document_id;
+end;
+$$;
+
create or replace function public.chunk_image_metadata(chunk_image_ids uuid[])
returns jsonb
language sql
@@ -409,6 +554,11 @@ as $$
'searchable', i.searchable,
'clinical_relevance_score', i.clinical_relevance_score,
'source_kind', i.source_kind,
+ 'sourceKind', i.source_kind,
+ 'tableLabel', nullif(i.metadata->>'table_label', ''),
+ 'tableTitle', nullif(i.metadata->>'table_title', ''),
+ 'tableRole', nullif(i.metadata->>'table_role', ''),
+ 'tableTextSnippet', nullif(left(coalesce(i.metadata->>'table_text_snippet', i.metadata->>'table_text', ''), 500), ''),
'width', i.width,
'height', i.height,
'labels', i.labels,
@@ -493,6 +643,7 @@ returns table (
similarity double precision,
text_rank double precision,
hybrid_score double precision,
+ rrf_score double precision,
images jsonb
)
language sql
@@ -502,7 +653,7 @@ as $$
with query as (
select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
),
- vector_candidates as (
+ vector_ranked as (
select
c.id,
c.document_id,
@@ -515,7 +666,9 @@ as $$
(
ts_rank_cd(c.search_tsv, query.tsq) +
(ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
- )::double precision as text_rank
+ )::double precision as text_rank,
+ row_number() over (order by c.embedding <=> query_embedding) as vector_rank,
+ null::bigint as text_match_rank
from public.document_chunks c
join public.documents d on d.id = c.document_id
cross join query
@@ -525,7 +678,7 @@ as $$
order by c.embedding <=> query_embedding
limit greatest(match_count * 6, 48)
),
- text_candidates as (
+ text_ranked as (
select
c.id,
c.document_id,
@@ -538,7 +691,16 @@ as $$
(
ts_rank_cd(c.search_tsv, query.tsq) +
(ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
- )::double precision as text_rank
+ )::double precision as text_rank,
+ null::bigint as vector_rank,
+ row_number() over (
+ order by
+ (
+ ts_rank_cd(c.search_tsv, query.tsq) +
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0)
+ ) desc,
+ c.embedding <=> query_embedding
+ ) as text_match_rank
from public.document_chunks c
join public.documents d on d.id = c.document_id
cross join query
@@ -552,9 +714,25 @@ as $$
limit greatest(match_count * 6, 48)
),
combined as (
- select * from vector_candidates
- union
- select * from text_candidates
+ select * from vector_ranked
+ union all
+ select * from text_ranked
+ ),
+ scored as (
+ select
+ id,
+ document_id,
+ page_number,
+ chunk_index,
+ section_heading,
+ content,
+ image_ids,
+ max(similarity)::double precision as similarity,
+ max(text_rank)::double precision as text_rank,
+ min(vector_rank) as vector_rank,
+ min(text_match_rank) as text_match_rank
+ from combined
+ group by id, document_id, page_number, chunk_index, section_heading, content, image_ids
)
select
c.id,
@@ -570,13 +748,209 @@ as $$
c.similarity,
c.text_rank,
((c.similarity * 0.72) + (least(c.text_rank, 1) * 0.28))::double precision as hybrid_score,
+ (
+ coalesce(1.0 / (60 + c.vector_rank), 0) +
+ coalesce(1.0 / (60 + c.text_match_rank), 0)
+ )::double precision as rrf_score,
public.chunk_image_metadata(c.image_ids) as images
- from combined c
+ from scored c
join public.documents d on d.id = c.document_id
order by hybrid_score desc, c.similarity desc, c.text_rank desc
limit match_count;
$$;
+create or replace function public.match_document_memory_cards_hybrid(
+ query_embedding vector(1536),
+ query_text text,
+ match_count integer default 32,
+ min_similarity double precision default 0.1,
+ document_filters uuid[] default null,
+ owner_filter uuid default null
+)
+returns table (
+ id uuid,
+ document_id uuid,
+ owner_id uuid,
+ section_id uuid,
+ card_type text,
+ title text,
+ content text,
+ normalized_terms text[],
+ page_number integer,
+ source_chunk_ids uuid[],
+ source_image_ids uuid[],
+ confidence real,
+ metadata jsonb,
+ similarity double precision,
+ text_rank double precision,
+ hybrid_score double precision,
+ rrf_score double precision
+)
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ with query as (
+ select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
+ ),
+ vector_ranked as (
+ select
+ m.*,
+ 1 - (m.embedding <=> query_embedding) as similarity,
+ ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank,
+ row_number() over (order by m.embedding <=> query_embedding) as vector_rank,
+ null::bigint as text_match_rank
+ from public.document_memory_cards m
+ join public.documents d on d.id = m.document_id
+ cross join query
+ where (document_filters is null or m.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and 1 - (m.embedding <=> query_embedding) >= min_similarity
+ order by m.embedding <=> query_embedding
+ limit greatest(match_count * 4, 64)
+ ),
+ text_ranked as (
+ select
+ m.*,
+ 1 - (m.embedding <=> query_embedding) as similarity,
+ ts_rank_cd(m.search_tsv, query.tsq)::double precision as text_rank,
+ null::bigint as vector_rank,
+ row_number() over (
+ order by ts_rank_cd(m.search_tsv, query.tsq) desc, m.embedding <=> query_embedding
+ ) as text_match_rank
+ from public.document_memory_cards m
+ join public.documents d on d.id = m.document_id
+ cross join query
+ where (document_filters is null or m.document_id = any(document_filters))
+ and (owner_filter is null or d.owner_id = owner_filter)
+ and m.search_tsv @@ query.tsq
+ order by ts_rank_cd(m.search_tsv, query.tsq) desc
+ limit greatest(match_count * 4, 64)
+ ),
+ combined as (
+ select * from vector_ranked
+ union all
+ select * from text_ranked
+ ),
+ scored as (
+ select
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata,
+ max(similarity)::double precision as similarity,
+ max(text_rank)::double precision as text_rank,
+ min(vector_rank) as vector_rank,
+ min(text_match_rank) as text_match_rank
+ from combined
+ group by
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata
+ )
+ select
+ id,
+ document_id,
+ owner_id,
+ section_id,
+ card_type,
+ title,
+ content,
+ normalized_terms,
+ page_number,
+ source_chunk_ids,
+ source_image_ids,
+ confidence,
+ metadata,
+ similarity,
+ text_rank,
+ ((similarity * 0.65) + (least(text_rank, 1) * 0.25) + (confidence * 0.10))::double precision as hybrid_score,
+ (
+ coalesce(1.0 / (60 + vector_rank), 0) +
+ coalesce(1.0 / (60 + text_match_rank), 0)
+ )::double precision as rrf_score
+ from scored
+ order by hybrid_score desc, similarity desc, text_rank desc, confidence desc
+ limit match_count;
+$$;
+
+create or replace function public.match_documents_for_query(
+ query_text text,
+ match_count integer default 12,
+ owner_filter uuid default null
+)
+returns table (
+ id uuid,
+ owner_id uuid,
+ title text,
+ file_name text,
+ status text,
+ page_count integer,
+ chunk_count integer,
+ image_count integer,
+ metadata jsonb,
+ text_rank double precision,
+ match_reason text
+)
+language sql
+stable
+set search_path = public, extensions, pg_temp
+as $$
+ with query as (
+ select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq
+ ),
+ ranked as (
+ select
+ d.id,
+ d.owner_id,
+ d.title,
+ d.file_name,
+ d.status,
+ d.page_count,
+ d.chunk_count,
+ d.image_count,
+ d.metadata,
+ (
+ (ts_rank_cd(d.title_search_tsv, query.tsq) * 4.0) +
+ (ts_rank_cd(d.search_tsv, query.tsq) * 1.5)
+ )::double precision as text_rank,
+ case
+ when d.title_search_tsv @@ query.tsq then 'title'
+ when d.search_tsv @@ query.tsq then 'metadata'
+ else 'none'
+ end as match_reason
+ from public.documents d
+ cross join query
+ where (owner_filter is null or d.owner_id = owner_filter)
+ and d.status = 'indexed'
+ and (d.title_search_tsv @@ query.tsq or d.search_tsv @@ query.tsq)
+ )
+ select *
+ from ranked
+ where text_rank > 0
+ order by text_rank desc, page_count desc, title asc
+ limit match_count;
+$$;
+
create or replace function public.match_document_chunks_text(
query_text text,
match_count integer default 12,
@@ -705,8 +1079,26 @@ as $$
and (owner_filter is null or d.owner_id = owner_filter);
$$;
+alter default privileges for role postgres in schema public
+ revoke all privileges on tables from anon, authenticated;
+alter default privileges for role postgres in schema public
+ revoke usage, select on sequences from anon, authenticated;
+alter default privileges for role postgres in schema public
+ revoke execute on functions from public, anon, authenticated;
+alter default privileges for role postgres in schema public
+ grant select, insert, update, delete on tables to service_role;
+alter default privileges for role postgres in schema public
+ grant usage, select on sequences to service_role;
+alter default privileges for role postgres in schema public
+ grant execute on functions to service_role;
+
+revoke usage on schema public from anon;
grant usage on schema public to authenticated, service_role;
+revoke all privileges on all tables in schema public from anon, authenticated;
+revoke all privileges on all sequences in schema public from anon, authenticated;
+revoke execute on all functions in schema public from public, anon, authenticated;
+
grant select, insert, update, delete on table
public.import_batches,
public.documents,
@@ -715,27 +1107,30 @@ grant select, insert, update, delete on table
public.image_caption_cache,
public.document_labels,
public.document_summaries,
+ public.document_sections,
+ public.document_memory_cards,
public.document_chunks,
public.ingestion_jobs,
- public.rag_queries
+ public.rag_queries,
+ public.storage_cleanup_jobs
to service_role;
grant usage, select on all sequences in schema public to service_role;
grant execute on all functions in schema public to service_role;
-grant execute on function public.claim_ingestion_jobs(text, integer, integer) to service_role;
-grant execute on function public.reset_document_index(uuid) to service_role;
-grant select on table public.import_batches to authenticated;
-grant select, insert, update, delete on table public.documents to authenticated;
grant select on table
+ public.import_batches,
+ public.documents,
public.document_pages,
public.document_images,
public.document_labels,
public.document_summaries,
public.document_chunks,
- public.ingestion_jobs
+ public.ingestion_jobs,
+ public.rag_queries,
+ public.storage_cleanup_jobs
to authenticated;
-grant select, insert on table public.rag_queries to authenticated;
+
grant insert, update, delete on table public.document_labels to authenticated;
alter table public.import_batches enable row level security;
@@ -745,21 +1140,18 @@ alter table public.document_images enable row level security;
alter table public.image_caption_cache enable row level security;
alter table public.document_labels enable row level security;
alter table public.document_summaries enable row level security;
+alter table public.document_sections enable row level security;
+alter table public.document_memory_cards enable row level security;
alter table public.document_chunks enable row level security;
alter table public.ingestion_jobs enable row level security;
alter table public.rag_queries enable row level security;
+alter table public.storage_cleanup_jobs enable row level security;
create policy "import batches owner read" on public.import_batches
for select to authenticated using (owner_id = (select auth.uid()));
create policy "documents owner read" on public.documents
for select to authenticated using (owner_id = (select auth.uid()));
-create policy "documents owner insert" on public.documents
- for insert to authenticated with check (owner_id = (select auth.uid()));
-create policy "documents owner update" on public.documents
- for update to authenticated using (owner_id = (select auth.uid())) with check (owner_id = (select auth.uid()));
-create policy "documents owner delete" on public.documents
- for delete to authenticated using (owner_id = (select auth.uid()));
create policy "pages owner read" on public.document_pages
for select to authenticated using (
@@ -797,8 +1189,9 @@ create policy "jobs owner read" on public.ingestion_jobs
create policy "rag owner read" on public.rag_queries
for select to authenticated using (owner_id = (select auth.uid()));
-create policy "rag owner insert" on public.rag_queries
- for insert to authenticated with check (owner_id = (select auth.uid()));
+
+create policy "storage cleanup owner read" on public.storage_cleanup_jobs
+ for select to authenticated using (owner_id = (select auth.uid()));
create policy "document storage owner read" on storage.objects
for select to authenticated
diff --git a/tests/accessible-table-normalization.test.ts b/tests/accessible-table-normalization.test.ts
new file mode 100644
index 000000000..f3d602f42
--- /dev/null
+++ b/tests/accessible-table-normalization.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, it } from "vitest";
+import { normalizeAccessibleTable } from "../src/lib/accessible-table-normalization";
+
+describe("normalizeAccessibleTable", () => {
+ it("drops empty OCR spacer columns instead of rendering generic column names", () => {
+ const table = normalizeAccessibleTable([
+ [
+ "Code/ description",
+ "",
+ "",
+ "Response type/",
+ "Typical presentations",
+ "",
+ "",
+ "MH Service action/ response",
+ "",
+ "",
+ "Additional actions to be considered",
+ ],
+ ["", "", "", "time to face-to-", "", "", "", "", "", "", ""],
+ [
+ "A Current actions endangering self or others",
+ "",
+ "",
+ "Emergency services response IMMEDIATE REFERRAL",
+ "Overdose. Other medical emergency.",
+ "",
+ "",
+ "Community clinician notify ambulance, police or fire brigade.",
+ "",
+ "",
+ "Keeping caller on line until emergency services arrive.",
+ ],
+ ]);
+
+ expect(table?.header).toEqual([
+ "Code/ description",
+ "Response type/ time to face-to-",
+ "Typical presentations",
+ "MH Service action/ response",
+ "Additional actions to be considered",
+ ]);
+ expect(table?.header.join(" ")).not.toMatch(/Column \d/i);
+ expect(table?.body[0]?.[3]).toContain("Community clinician notify ambulance");
+ });
+
+ it("merges sparse unnamed continuation columns and rows into the nearest real column", () => {
+ const table = normalizeAccessibleTable([
+ [
+ "Code/ description",
+ "",
+ "",
+ "Response type/",
+ "Typical presentations",
+ "",
+ "",
+ "MH Service action/ response",
+ "",
+ "",
+ "Additional actions to be considered",
+ ],
+ [
+ "C High risk of harm to self or others",
+ "",
+ "",
+ "Urgent MH response WITHIN 12 HOURS",
+ "",
+ "Suicidal ideation with no plan",
+ "",
+ "Community clinician face-to-face assessment within 8 hours",
+ "",
+ "",
+ "As above",
+ ],
+ ["", "", "", "", "", "and/or history of suicidal", "", "", "", "", ""],
+ ["", "", "", "", "", "ideation", "", "", "", "", ""],
+ [
+ "",
+ "F",
+ "",
+ "Requires further triage contact/ follow up",
+ "",
+ "Other service more appropriate",
+ "",
+ "",
+ "Community clinician to",
+ "",
+ "Facilitating appointment with alternative provider",
+ ],
+ ["", "Referral: not requiring face-to-face response", "", "", "", "", "", "", "provide formal referral", "", ""],
+ ]);
+
+ expect(table?.body[0]).toEqual([
+ "C High risk of harm to self or others",
+ "Urgent MH response WITHIN 12 HOURS",
+ "Suicidal ideation with no plan and/or history of suicidal ideation",
+ "Community clinician face-to-face assessment within 8 hours",
+ "As above",
+ ]);
+ expect(table?.body[1]).toEqual([
+ "F Referral: not requiring face-to-face response",
+ "Requires further triage contact/ follow up",
+ "Other service more appropriate",
+ "Community clinician to provide formal referral",
+ "Facilitating appointment with alternative provider",
+ ]);
+ });
+});
diff --git a/tests/answer-formatting.test.ts b/tests/answer-formatting.test.ts
new file mode 100644
index 000000000..5d22cd630
--- /dev/null
+++ b/tests/answer-formatting.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+import { answerLinePresentation, parseAnswerDisplayContent } from "../src/lib/answer-formatting";
+
+describe("answer display formatting", () => {
+ it("parses markdown-style answer bullets into labeled display rows", () => {
+ const parsed = parseAnswerDisplayContent(
+ "- Monitoring: Check observations after IM medication.\n- Dose detail: Lorazepam **1mg** to **2mg** may be used when supported.",
+ );
+
+ expect(parsed.type).toBe("bullets");
+ expect(parsed.mode).toBe("clinical_pathway");
+ expect(parsed.lines).toMatchObject([
+ { label: "Monitoring", text: "Check observations after IM medication." },
+ { label: "Dose detail", text: "Lorazepam **1mg** to **2mg** may be used when supported." },
+ ]);
+ expect(answerLinePresentation(parsed.lines[0])).toMatchObject({ tone: "monitoring", symbol: "⏱" });
+ expect(answerLinePresentation(parsed.lines[1])).toMatchObject({ tone: "medication", symbol: "Rx" });
+ });
+
+ it("recovers inline bullets after whitespace has been compacted", () => {
+ const parsed = parseAnswerDisplayContent(
+ "- Bottom line: Use the source-backed pathway. - Escalation/risk: Seek senior review if no response.",
+ );
+
+ expect(parsed.type).toBe("bullets");
+ expect(parsed.mode).toBe("clinical_pathway");
+ expect(parsed.lines).toHaveLength(2);
+ expect(parsed.lines[0].label).toBe("Bottom line");
+ expect(parsed.lines[1].label).toBe("Escalation/risk");
+ expect(answerLinePresentation(parsed.lines[0])).toMatchObject({ tone: "direct", symbol: "✓" });
+ expect(answerLinePresentation(parsed.lines[1])).toMatchObject({ tone: "risk", symbol: "!" });
+ });
+
+ it("keeps ordinary prose as a paragraph", () => {
+ const parsed = parseAnswerDisplayContent("The indexed source does not contain enough information.");
+
+ expect(parsed.type).toBe("paragraph");
+ expect(parsed.mode).toBe("evidence_gap");
+ expect(parsed.lines[0]).toMatchObject({ label: null, text: "The indexed source does not contain enough information." });
+ expect(answerLinePresentation(parsed.lines[0])).toMatchObject({ tone: "gap", symbol: "?" });
+ });
+
+ it("uses checklist mode for practical action answers", () => {
+ const parsed = parseAnswerDisplayContent(
+ "- Required actions: Complete the source form.\n- Documentation/forms: Record review and consent.",
+ );
+
+ expect(parsed.mode).toBe("checklist");
+ expect(answerLinePresentation(parsed.lines[0])).toMatchObject({ tone: "action", symbol: "→" });
+ expect(answerLinePresentation(parsed.lines[1])).toMatchObject({ tone: "documentation", symbol: "§" });
+ });
+
+ it("uses comparison mode and symbols for contrast answers", () => {
+ const parsed = parseAnswerDisplayContent(
+ "- Comparison: One document describes routine monitoring.\n- Source point: Another source describes escalation criteria.",
+ );
+
+ expect(parsed.mode).toBe("comparison");
+ expect(answerLinePresentation(parsed.lines[0])).toMatchObject({ tone: "comparison", symbol: "↔" });
+ expect(answerLinePresentation(parsed.lines[1])).toMatchObject({ tone: "source", symbol: "#" });
+ });
+});
diff --git a/tests/answer-ranking.test.ts b/tests/answer-ranking.test.ts
new file mode 100644
index 000000000..75a90ac08
--- /dev/null
+++ b/tests/answer-ranking.test.ts
@@ -0,0 +1,185 @@
+import { describe, expect, it } from "vitest";
+import { boldHighYieldClinicalText, rankAnswerEvidence } from "../src/lib/answer-ranking";
+import { parseAnswerJson } from "../src/lib/rag";
+import type { SearchResult } from "../src/lib/types";
+
+function result(overrides: Partial = {}): SearchResult {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ title: overrides.title ?? "Guideline",
+ file_name: overrides.file_name ?? "guideline.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: overrides.section_heading ?? null,
+ content: overrides.content ?? "General clinical source text.",
+ image_ids: [],
+ similarity: overrides.similarity ?? 0.6,
+ hybrid_score: overrides.hybrid_score ?? 0.6,
+ images: [],
+ ...overrides,
+ };
+}
+
+describe("answer evidence ranking", () => {
+ it("promotes a directly answering lower-score chunk over a broad higher-score chunk", () => {
+ const ranking = rankAnswerEvidence("What ANC threshold should withhold clozapine?", [
+ result({
+ id: "broad-high-score",
+ title: "Clozapine Prescribing",
+ content: "Clozapine service appointments, consent, and general monitoring responsibilities.",
+ hybrid_score: 0.9,
+ }),
+ result({
+ id: "direct-lower-score",
+ title: "Clozapine Prescribing",
+ section_heading: "FBC and ANC monitoring",
+ content: "Withhold clozapine when ANC or FBC thresholds require treatment interruption and urgent review.",
+ hybrid_score: 0.62,
+ }),
+ ]);
+
+ expect(ranking.rankedResults[0].id).toBe("direct-lower-score");
+ expect(ranking.topScore).toBeGreaterThan(0.6);
+ });
+
+ it("prioritizes table-heavy evidence for table and threshold questions", () => {
+ const ranking = rankAnswerEvidence("Which table covers agitation and arousal pharmacological management?", [
+ result({
+ id: "plain-text",
+ title: "Agitation notes",
+ content: "Agitation and arousal are mentioned in background notes.",
+ hybrid_score: 0.72,
+ }),
+ result({
+ id: "table-evidence",
+ title: "Agitation and Arousal Pharmacological Management",
+ content: "Table text lists rating 2-3, oral medication, intramuscular medication, and escalation steps.",
+ hybrid_score: 0.58,
+ images: [
+ {
+ id: "image-1",
+ page_number: 2,
+ storage_path: "private/image.png",
+ caption: "Agitation and arousal pharmacological management table.",
+ image_type: "clinical_table",
+ searchable: true,
+ sourceKind: "table_crop",
+ },
+ ],
+ }),
+ ]);
+
+ expect(ranking.rankedResults[0].id).toBe("table-evidence");
+ });
+
+ it("supports medication-risk, document-lookup, broad-summary, and unsupported-style cases", () => {
+ expect(
+ rankAnswerEvidence("How should oral medication dose be managed?", [
+ result({ id: "generic", content: "General ward process.", hybrid_score: 0.7 }),
+ result({
+ id: "dose",
+ content: "Medication dose, route, oral administration, and monitoring timing are listed.",
+ hybrid_score: 0.55,
+ }),
+ ]).rankedResults[0].id,
+ ).toBe("dose");
+
+ expect(
+ rankAnswerEvidence("Find the NOCC document", [
+ result({ id: "generic", title: "Generic Mental Health Document", hybrid_score: 0.7 }),
+ result({ id: "nocc", title: "MHSP.NOCC", file_name: "MHSP.NOCC.pdf", hybrid_score: 0.5 }),
+ ]).rankedResults[0].id,
+ ).toBe("nocc");
+
+ expect(
+ rankAnswerEvidence("Summarize discharge guidance", [
+ result({ id: "generic", content: "Administrative content.", hybrid_score: 0.68 }),
+ result({
+ id: "summary-match",
+ title: "Discharge",
+ document_summary: "Discharge guidance, documentation, follow-up and clinical responsibilities.",
+ content: "Discharge procedure overview.",
+ hybrid_score: 0.5,
+ }),
+ ]).rankedResults[0].id,
+ ).toBe("summary-match");
+
+ expect(
+ rankAnswerEvidence("What is the diabetic ketoacidosis insulin protocol?", [
+ result({ id: "nearby", content: "Long acting injectable antipsychotic appointment process.", hybrid_score: 0.33 }),
+ ]).topScore,
+ ).toBeLessThan(0.45);
+ });
+
+ it("does not let agitation title repetition outrank direct dosing evidence", () => {
+ const ranking = rankAnswerEvidence("agitation and arousal dosing in psychiatric patients", [
+ result({
+ id: "title-repeat",
+ title: "Agitation and Arousal Pharmacological Management Guideline",
+ content:
+ "Agitation and Arousal: Pharmacological Management Guideline - Agitation and Arousal pharmacological management for adult mental health inpatients.",
+ hybrid_score: 0.94,
+ }),
+ result({
+ id: "dosing-table",
+ title: "Agitation and Arousal Pharmacological Management Guideline",
+ section_heading: "Medication dose details",
+ content:
+ "Medication options include oral olanzapine or lorazepam, intramuscular medication when oral options are not appropriate, dose escalation limits, and monitoring after administration.",
+ hybrid_score: 0.62,
+ }),
+ ]);
+
+ expect(ranking.rankedResults[0].id).toBe("dosing-table");
+ });
+});
+
+describe("high-yield answer bolding", () => {
+ it("bolds high-yield clinical details without double-bolding existing markdown", () => {
+ const formatted = boldHighYieldClinicalText(
+ "Withhold clozapine when FBC is unsafe and repeat review after 4 hours. Existing **ANC** stays stable.",
+ "What FBC threshold should withhold clozapine?",
+ );
+
+ expect(formatted).toContain("**Withhold**");
+ expect(formatted).toContain("**clozapine**");
+ expect(formatted).toContain("**FBC**");
+ expect(formatted).toContain("**4 hours**");
+ expect(formatted).toContain("Existing **ANC** stays stable.");
+ expect(formatted).not.toContain("****ANC****");
+ });
+
+ it("applies high-yield bolding to structured model answers and sections", () => {
+ const answer = parseAnswerJson(
+ JSON.stringify({
+ answer: "Withhold clozapine when FBC is unsafe.",
+ grounded: true,
+ confidence: "high",
+ answerSections: [
+ {
+ heading: "Escalation",
+ body: "Urgent review is required within 4 hours.",
+ citation_chunk_ids: ["chunk-1"],
+ },
+ ],
+ citations: [{ chunk_id: "chunk-1" }],
+ quoteCards: [],
+ conflictsOrGaps: [],
+ }),
+ [
+ result({
+ id: "chunk-1",
+ content: "Withhold clozapine when FBC is unsafe and arrange urgent review within 4 hours.",
+ similarity: 0.9,
+ }),
+ ],
+ "What FBC threshold should withhold clozapine?",
+ );
+
+ expect(answer.answer).toContain("**Withhold**");
+ expect(answer.answer).toContain("**clozapine**");
+ expect(answer.answer).toContain("**FBC**");
+ expect(answer.answerSections?.[0]?.body).toContain("**4 hours**");
+ });
+});
diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts
index bc8e23185..807d34abb 100644
--- a/tests/chunking.test.ts
+++ b/tests/chunking.test.ts
@@ -38,6 +38,25 @@ describe("image-aware chunks", () => {
expect(tag).toContain("[[IMAGE_DATA_END]]");
});
+ it("preserves table labels, titles, and extracted table text as searchable context", () => {
+ const tag = buildImageTag({
+ id: "table-1",
+ caption: "Agitation management table.",
+ imageType: "clinical_table",
+ sourceKind: "table_crop",
+ tableLabel: "Table 1",
+ tableTitle: "Agitation and arousal rating scale and associated management",
+ tableRole: "clinical",
+ tableTextSnippet: "Score 5 | Highly aroused and violent toward others and/or property",
+ });
+
+ expect(tag).toContain("Source kind: table_crop");
+ expect(tag).toContain("Table label: Table 1");
+ expect(tag).toContain("Table role: clinical");
+ expect(tag).toContain("Agitation and arousal rating scale");
+ expect(tag).toContain("Score 5");
+ });
+
it("attaches referenced image ids to chunks", () => {
const chunks = buildChunks([
{
diff --git a/tests/clinical-safety.test.ts b/tests/clinical-safety.test.ts
index 570e12808..7fd58f33d 100644
--- a/tests/clinical-safety.test.ts
+++ b/tests/clinical-safety.test.ts
@@ -24,6 +24,30 @@ const answer: RagAnswer = {
],
};
+const directRelevance = {
+ verdict: "direct" as const,
+ label: "Direct match",
+ matchedTerms: ["urgent"],
+ missingTerms: [],
+ directSourceCount: 1,
+ weakSourceCount: 0,
+ score: 0.9,
+ supportReason: "Direct indexed support found.",
+ isSourceBacked: true,
+};
+
+const nearbyRelevance = {
+ verdict: "nearby" as const,
+ label: "Nearby only",
+ matchedTerms: ["monitoring"],
+ missingTerms: ["lithium"],
+ directSourceCount: 0,
+ weakSourceCount: 1,
+ score: 0.32,
+ supportReason: "Only nearby indexed passages were found.",
+ isSourceBacked: false,
+};
+
describe("clinical safety findings", () => {
it("extracts only source-backed safety findings from grounded answers", () => {
const findings = extractSafetyFindings(answer);
@@ -36,4 +60,44 @@ describe("clinical safety findings", () => {
it("does not show safety findings for ungrounded answers", () => {
expect(extractSafetyFindings({ ...answer, grounded: false })).toEqual([]);
});
+
+ it("suppresses generic safety findings when evidence is nearby only", () => {
+ expect(extractSafetyFindings({ ...answer, relevance: nearbyRelevance })).toEqual([]);
+ });
+
+ it("keeps safety findings when relevance is source-backed", () => {
+ const findings = extractSafetyFindings({
+ ...answer,
+ relevance: directRelevance,
+ sources: answer.sources.map((source) => ({ ...source, source_strength: "moderate" })),
+ });
+
+ expect(findings).toHaveLength(1);
+ expect(findings[0].label).toBe("Red flag");
+ });
+
+ it("does not leak internal image or table metadata in safety findings", () => {
+ const findings = extractSafetyFindings({
+ ...answer,
+ quoteCards: [
+ {
+ chunk_id: "chunk-1",
+ document_id: "doc-1",
+ title: "Risk source",
+ file_name: "risk.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: null,
+ quote:
+ "[[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Image type: clinical_table; Table role: clinical; Table text: | Dose | Route | [[IMAGE_DATA_END]] Monitor blood tests after dose changes.",
+ },
+ ],
+ sources: [],
+ });
+
+ expect(findings[0].text).toContain("Monitor blood tests");
+ expect(findings[0].text).not.toContain("[[IMAGE_DATA_START]]");
+ expect(findings[0].text).not.toContain("Image ID:");
+ expect(findings[0].text).not.toContain("Table text:");
+ });
});
diff --git a/tests/clinical-search.test.ts b/tests/clinical-search.test.ts
index a8e079952..3b535a6ff 100644
--- a/tests/clinical-search.test.ts
+++ b/tests/clinical-search.test.ts
@@ -1,7 +1,45 @@
import { describe, expect, it } from "vitest";
-import { buildClinicalTextSearchQuery, normalizedClinicalSearchTokens } from "../src/lib/clinical-search";
+import {
+ buildClinicalTextSearchQuery,
+ classifyRagQuery,
+ clinicalRankExplanation,
+ normalizedClinicalSearchTokens,
+ rankClinicalResults,
+} from "../src/lib/clinical-search";
+import type { SearchResult } from "../src/lib/types";
+
+function result(overrides: Partial): SearchResult {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ title: overrides.title ?? "Guideline",
+ file_name: overrides.file_name ?? "guideline.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: null,
+ content: overrides.content ?? "Treatment process text.",
+ image_ids: [],
+ similarity: overrides.similarity ?? 0.6,
+ hybrid_score: overrides.hybrid_score ?? 0.6,
+ images: [],
+ ...overrides,
+ };
+}
describe("clinical search query normalization", () => {
+ it("classifies common RAG query shapes for routing and observability", () => {
+ expect(classifyRagQuery("Find the NOCC document").queryClass).toBe("document_lookup");
+ expect(classifyRagQuery("What ANC threshold should stop clozapine?").queryClass).toBe("table_threshold");
+ expect(classifyRagQuery("How are long acting injectable medications managed?").queryClass).toBe(
+ "medication_dose_risk",
+ );
+ expect(classifyRagQuery("agitation and arousal dosing in psychiatric patients").queryClass).toBe(
+ "medication_dose_risk",
+ );
+ expect(classifyRagQuery("Compare admission and discharge requirements").queryClass).toBe("comparison");
+ expect(classifyRagQuery("Summarize the discharge guidance").queryClass).toBe("broad_summary");
+ });
+
it("keeps high-yield clinical terms and removes question filler", () => {
expect(normalizedClinicalSearchTokens("What safety monitoring is required for clozapine?")).toEqual([
"safety",
@@ -14,6 +52,9 @@ describe("clinical search query normalization", () => {
expect(buildClinicalTextSearchQuery("What antibiotic dose is recommended for community-acquired pneumonia?")).toBe(
"antibiotic dose recommended community acquired pneumonia",
);
+ expect(buildClinicalTextSearchQuery("Please can you find agitation and arousal dosing for me?")).toBe(
+ "agitation arousal dosing",
+ );
});
it("falls back to the original query when only one useful token remains", () => {
@@ -37,4 +78,193 @@ describe("clinical search query normalization", () => {
"illegal substance",
);
});
+
+ it("removes table-coverage filler while keeping the clinical topic", () => {
+ expect(buildClinicalTextSearchQuery("Which table covers agitation and arousal pharmacological management?")).toBe(
+ "table agitation arousal pharmacological",
+ );
+ });
+
+ it("boosts exact treatment team process title matches above broader treatment-process hits", () => {
+ const ranked = rankClinicalResults("What is the mental health treatment team process?", [
+ result({
+ id: "assessment-treatment",
+ title: "MHSP.MHATT.AssessmentTreatmentProcess",
+ file_name: "MHSP.MHATT.AssessmentTreatmentProcess.pdf",
+ hybrid_score: 0.65,
+ }),
+ result({
+ id: "treatment-team",
+ title: "MHSP.MHAT.MHCT.TreatmentTeamProcess",
+ file_name: "MHSP.MHAT.MHCT.TreatmentTeamProcess.pdf",
+ hybrid_score: 0.61,
+ }),
+ ]);
+
+ expect(ranked[0].id).toBe("treatment-team");
+ });
+
+ it("keeps lookup scoring path stable when section and content have clinical tokens", () => {
+ const searchQuery = "document lookup section page 3 treatment team review";
+ const ranked = rankClinicalResults(searchQuery, [
+ result({
+ id: "lookup-match",
+ title: "Clozapine Prescribing and Monitoring",
+ file_name: "clozapine-prescribing.pdf",
+ section_heading: "Section 3: Safety Monitoring",
+ content: "Monitoring requirements and safety thresholds are listed by section and page.",
+ hybrid_score: 0.61,
+ }),
+ result({
+ id: "unrelated",
+ title: "Generic Mental Health Notes",
+ file_name: "notes.docx",
+ section_heading: "Overview",
+ content: "General team process and broad guidance references.",
+ hybrid_score: 0.75,
+ }),
+ ]);
+
+ expect(ranked).toHaveLength(2);
+ expect(ranked.map((item) => item.id).sort()).toEqual(["lookup-match", "unrelated"]);
+ });
+
+ it("uses generated labels and summaries as deterministic ranking signals", () => {
+ const ranked = rankClinicalResults("Find the metabolic monitoring document", [
+ result({
+ id: "metadata-match",
+ title: "General Monitoring",
+ file_name: "general-monitoring.pdf",
+ content: "General review text.",
+ hybrid_score: 0.6,
+ document_labels: [
+ {
+ id: "label-1",
+ document_id: "doc-1",
+ label: "metabolic monitoring",
+ label_type: "topic",
+ source: "generated",
+ confidence: 0.91,
+ },
+ ],
+ document_summary: "Metabolic monitoring requirements and review timing.",
+ }),
+ result({
+ id: "higher-base",
+ title: "Generic Monitoring",
+ file_name: "generic-monitoring.pdf",
+ content: "General monitoring review without specialty specifics.",
+ hybrid_score: 0.66,
+ }),
+ ]);
+
+ expect(ranked[0].id).toBe("metadata-match");
+ });
+
+ it("ranks arbitrary newly uploaded documents from generic labels and summaries", () => {
+ const ranked = rankClinicalResults("future protocol escalation pathway", [
+ result({
+ id: "new-upload",
+ title: "Ward Reference Pack",
+ file_name: "new-upload.pdf",
+ content: "General ward reference notes.",
+ hybrid_score: 0.55,
+ document_labels: [
+ {
+ id: "label-future",
+ document_id: "doc-future",
+ label: "escalation pathway",
+ label_type: "workflow",
+ source: "generated",
+ confidence: 0.93,
+ },
+ ],
+ document_summary: "Future protocol escalation pathway for clinical workflow decisions.",
+ }),
+ result({
+ id: "generic",
+ title: "Administrative Pack",
+ file_name: "generic.pdf",
+ content: "General administrative checklist.",
+ hybrid_score: 0.66,
+ }),
+ ]);
+
+ expect(ranked[0].id).toBe("new-upload");
+ });
+
+ it("still ranks source chunks when enrichment labels and summaries are absent", () => {
+ const ranked = rankClinicalResults("observation interval after medication change", [
+ result({
+ id: "chunk-match",
+ title: "Recently Uploaded Guideline",
+ file_name: "recent-upload.pdf",
+ content: "After a medication change, the observation interval must be reviewed and documented.",
+ hybrid_score: 0.58,
+ }),
+ result({
+ id: "metadata-absent-unrelated",
+ title: "Unrelated Uploaded Guideline",
+ file_name: "unrelated-upload.pdf",
+ content: "Discharge appointment administration and filing process.",
+ hybrid_score: 0.62,
+ }),
+ ]);
+
+ expect(ranked[0].id).toBe("chunk-match");
+ });
+
+ it("attaches score explanations while keeping weighted hybrid ranking as the served default", () => {
+ const ranked = rankClinicalResults("monitoring requirements", [
+ result({
+ id: "weighted-winner",
+ title: "Monitoring Requirements",
+ file_name: "monitoring.pdf",
+ content: "Monitoring requirements are documented here.",
+ hybrid_score: 0.72,
+ similarity: 0.7,
+ text_rank: 0.4,
+ rrf_score: 0.01,
+ }),
+ result({
+ id: "rrf-only-contender",
+ title: "Monitoring Requirements",
+ file_name: "monitoring-alt.pdf",
+ content: "Monitoring requirements are documented here.",
+ hybrid_score: 0.64,
+ similarity: 0.62,
+ text_rank: 0.4,
+ rrf_score: 0.5,
+ }),
+ ]);
+
+ expect(ranked[0].id).toBe("weighted-winner");
+ expect(ranked[0].score_explanation).toMatchObject({
+ rrfScore: 0.01,
+ strategy: "weighted_hybrid_served_rrf_telemetry",
+ finalRank: 1,
+ });
+ });
+
+ it("produces stable score-explanation components", () => {
+ const explanation = clinicalRankExplanation(
+ "ANC threshold stop clozapine",
+ result({
+ title: "Clozapine Prescribing and Monitoring",
+ file_name: "clozapine.pdf",
+ content: "If ANC is below threshold, stop clozapine and urgently review monitoring.",
+ similarity: 0.71,
+ hybrid_score: 0.74,
+ text_rank: 0.6,
+ rrf_score: 0.2,
+ memory_score: 0.8,
+ }),
+ );
+
+ expect(explanation.vectorScore).toBe(0.71);
+ expect(explanation.weightedHybridScore).toBe(0.74);
+ expect(explanation.rrfScore).toBe(0.2);
+ expect(explanation.memoryBoost).toBeGreaterThan(0);
+ expect(explanation.finalScore).toBeGreaterThan(0.74);
+ });
});
diff --git a/tests/cross-document-synthesis.test.ts b/tests/cross-document-synthesis.test.ts
new file mode 100644
index 000000000..033b96459
--- /dev/null
+++ b/tests/cross-document-synthesis.test.ts
@@ -0,0 +1,104 @@
+import { describe, expect, it } from "vitest";
+import {
+ balanceCrossDocumentResults,
+ buildCrossDocumentFusionBrief,
+ buildCrossDocumentSynthesisPlan,
+ buildCrossDocumentSourceGuide,
+} from "../src/lib/cross-document-synthesis";
+import type { SearchResult } from "../src/lib/types";
+
+function source(overrides: Partial = {}): SearchResult {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ title: overrides.title ?? "Guideline",
+ file_name: overrides.file_name ?? "guideline.pdf",
+ page_number: overrides.page_number ?? 1,
+ chunk_index: overrides.chunk_index ?? 0,
+ section_heading: overrides.section_heading ?? null,
+ content: overrides.content ?? "Clinical guidance.",
+ image_ids: [],
+ similarity: overrides.similarity ?? 0.8,
+ hybrid_score: overrides.hybrid_score ?? 0.8,
+ images: [],
+ ...overrides,
+ };
+}
+
+describe("cross-document synthesis", () => {
+ it("balances source packing across documents before filling extra slots", () => {
+ const balanced = balanceCrossDocumentResults(
+ [
+ source({ id: "a1", document_id: "a", hybrid_score: 0.95 }),
+ source({ id: "a2", document_id: "a", hybrid_score: 0.93 }),
+ source({ id: "a3", document_id: "a", hybrid_score: 0.91 }),
+ source({ id: "b1", document_id: "b", hybrid_score: 0.74 }),
+ source({ id: "c1", document_id: "c", hybrid_score: 0.7 }),
+ ],
+ { limit: 4, maxPerDocument: 2, minDocuments: 3 },
+ );
+
+ expect(new Set(balanced.map((result) => result.document_id))).toEqual(new Set(["a", "b", "c"]));
+ expect(balanced.filter((result) => result.document_id === "a")).toHaveLength(2);
+ });
+
+ it("enables balanced packing for broad cross-document questions", () => {
+ const plan = buildCrossDocumentSynthesisPlan(
+ "What monitoring issues are important across these documents?",
+ [
+ source({ id: "a1", document_id: "a", title: "Lithium" }),
+ source({ id: "b1", document_id: "b", title: "Clozapine" }),
+ ],
+ "broad_summary",
+ );
+
+ expect(plan.enabled).toBe(true);
+ expect(plan.reason).toBe("broad_summary");
+ expect(plan.selectedDocumentCount).toBe(2);
+ });
+
+ it("leaves single-document answers untouched", () => {
+ const plan = buildCrossDocumentSynthesisPlan("Summarize this document", [source()], "broad_summary");
+
+ expect(plan.enabled).toBe(false);
+ expect(plan.results).toHaveLength(1);
+ });
+
+ it("builds a compact fused brief across selected documents", () => {
+ const results = [
+ source({
+ id: "lithium-1",
+ document_id: "lithium",
+ title: "Lithium Monitoring",
+ content: "Baseline renal and thyroid monitoring is required. Escalate for toxicity symptoms.",
+ }),
+ source({
+ id: "clozapine-1",
+ document_id: "clozapine",
+ title: "Clozapine Monitoring",
+ page_number: 4,
+ content: "FBC and ANC monitoring is required. Urgent review is needed for myocarditis symptoms.",
+ }),
+ ];
+
+ const brief = buildCrossDocumentFusionBrief("monitoring and escalation across documents", results);
+
+ expect(brief.documentCount).toBe(2);
+ expect(brief.bulletCount).toBe(2);
+ expect(brief.sourceChunkIds).toEqual(["lithium-1", "clozapine-1"]);
+ expect(brief.text).toContain("Fast fused source brief");
+ expect(brief.text).toContain("Lithium Monitoring");
+ expect(brief.text).toContain("Clozapine Monitoring");
+ });
+
+ it("builds a cross-document source guide with pages and chunk ids", () => {
+ const guide = buildCrossDocumentSourceGuide([
+ source({ id: "a1", document_id: "a", title: "A", page_number: 2 }),
+ source({ id: "b1", document_id: "b", title: "B", page_number: 5 }),
+ ]);
+
+ expect(guide).toContain("Cross-document synthesis guide");
+ expect(guide).toContain("A: use pages 2; source chunks a1");
+ expect(guide).toContain("B: use pages 5; source chunks b1");
+ });
+});
diff --git a/tests/deep-memory.test.ts b/tests/deep-memory.test.ts
new file mode 100644
index 000000000..e015624bd
--- /dev/null
+++ b/tests/deep-memory.test.ts
@@ -0,0 +1,251 @@
+import { describe, expect, it, vi } from "vitest";
+import { rankClinicalResults } from "../src/lib/clinical-search";
+import {
+ applyMemoryCardBoosts,
+ buildDocumentMemoryCards,
+ buildDocumentSections,
+ ragDeepMemoryVersion,
+ upsertDocumentDeepMemory,
+} from "../src/lib/deep-memory";
+import type { DocumentMemoryCard, SearchResult } from "../src/lib/types";
+
+vi.mock("@/lib/openai", () => ({
+ embedTexts: vi.fn(async (texts: string[]) => texts.map(() => Array.from({ length: 1536 }, () => 0.01))),
+}));
+
+const document = {
+ id: "doc-1",
+ owner_id: "user-1",
+ title: "Future Uploaded Clinical Protocol",
+ file_name: "future-upload.pdf",
+ source_path: "uploads/future-upload.pdf",
+ metadata: {},
+};
+
+function chunk(overrides: Partial & { chunk_index?: number; page_number?: number | null }) {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ page_number: overrides.page_number ?? 1,
+ chunk_index: overrides.chunk_index ?? 0,
+ section_heading: overrides.section_heading ?? "Clinical Workflow",
+ content: overrides.content ?? "Clinical protocol text.",
+ image_ids: overrides.image_ids ?? [],
+ metadata: overrides.source_metadata ?? {},
+ };
+}
+
+function result(overrides: Partial): SearchResult {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ title: overrides.title ?? "Future Uploaded Clinical Protocol",
+ file_name: overrides.file_name ?? "future-upload.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: "Clinical Workflow",
+ content: overrides.content ?? "Clinical protocol text.",
+ image_ids: [],
+ similarity: overrides.similarity ?? 0.55,
+ hybrid_score: overrides.hybrid_score ?? 0.55,
+ images: [],
+ ...overrides,
+ };
+}
+
+describe("deep RAG memory indexing", () => {
+ it("builds generic section maps from arbitrary uploaded document headings and page gaps", () => {
+ const sections = buildDocumentSections({
+ document,
+ chunks: [
+ chunk({ id: "chunk-a", chunk_index: 0, page_number: 1, section_heading: "Assessment" }),
+ chunk({ id: "chunk-b", chunk_index: 1, page_number: 2, section_heading: "Assessment" }),
+ chunk({ id: "chunk-c", chunk_index: 2, page_number: 5, section_heading: "Escalation" }),
+ ],
+ });
+
+ expect(sections).toHaveLength(2);
+ expect(sections[0]).toEqual(
+ expect.objectContaining({
+ heading: "Assessment",
+ page_start: 1,
+ page_end: 2,
+ chunk_ids: ["chunk-a", "chunk-b"],
+ }),
+ );
+ expect(sections[1]).toEqual(expect.objectContaining({ heading: "Escalation", chunk_ids: ["chunk-c"] }));
+ expect(sections[0].metadata).toEqual(expect.objectContaining({ rag_indexing_version: ragDeepMemoryVersion }));
+ });
+
+ it("extracts high-yield memory cards for tables, thresholds, risks, medications, and workflow steps", () => {
+ const cards = buildDocumentMemoryCards({
+ document,
+ chunks: [
+ chunk({
+ id: "chunk-table",
+ content:
+ "| Score | Action |\n| --- | --- |\n| 6 | Immediate senior review and lorazepam 1 mg PO. |\nIf ANC is < 1.5, stop clozapine and urgent specialist review is required.\nThe workflow requires documenting the review within 24 hours.",
+ }),
+ ],
+ });
+
+ expect(cards.map((card) => card.card_type)).toEqual(
+ expect.arrayContaining(["table_row", "threshold", "workflow"]),
+ );
+ expect(cards.some((card) => card.content.includes("lorazepam 1 mg"))).toBe(true);
+ expect(cards.every((card) => card.source_chunk_ids.includes("chunk-table") || card.card_type === "section_summary")).toBe(
+ true,
+ );
+ expect(cards.every((card) => card.metadata?.rag_indexing_version === ragDeepMemoryVersion)).toBe(true);
+ });
+
+ it("keeps a source-backed fallback card when no enrichment-style high-yield terms are present", () => {
+ const cards = buildDocumentMemoryCards({
+ document,
+ chunks: [
+ chunk({
+ id: "chunk-low-signal",
+ content: "This future upload describes local service background, scope, and routine contact information.",
+ }),
+ ],
+ });
+
+ expect(cards.length).toBeGreaterThan(0);
+ expect(cards.some((card) => card.source_chunk_ids.includes("chunk-low-signal"))).toBe(true);
+ });
+
+ it("does not cap persisted memory cards for large high-yield documents", () => {
+ const cards = buildDocumentMemoryCards({
+ document,
+ chunks: Array.from({ length: 140 }, (_, index) =>
+ chunk({
+ id: `chunk-risk-${index}`,
+ chunk_index: index,
+ page_number: index + 1,
+ content: `Risk workflow ${index}: urgent review is required within ${index + 1} hours and medication monitoring must be documented.`,
+ }),
+ ),
+ });
+
+ expect(cards.length).toBeGreaterThan(120);
+ expect(cards.some((card) => card.source_chunk_ids.includes("chunk-risk-139"))).toBe(true);
+ });
+
+ it("boosts direct memory evidence above a higher raw-score generic chunk", () => {
+ const boosted = applyMemoryCardBoosts(
+ "ANC threshold stop clozapine",
+ [
+ result({
+ id: "direct-threshold",
+ hybrid_score: 0.5,
+ content: "If ANC is < 1.5, stop clozapine and seek urgent specialist review.",
+ }),
+ result({
+ id: "generic-monitoring",
+ hybrid_score: 0.64,
+ content: "General monitoring should be documented in the clinical record.",
+ }),
+ ],
+ [
+ {
+ id: "card-1",
+ document_id: "doc-1",
+ owner_id: "user-1",
+ section_id: null,
+ card_type: "threshold",
+ title: "Threshold: ANC clozapine",
+ content: "If ANC is < 1.5, stop clozapine and seek urgent specialist review.",
+ normalized_terms: ["anc", "threshold", "stop", "clozapine"],
+ page_number: 1,
+ source_chunk_ids: ["direct-threshold"],
+ source_image_ids: [],
+ confidence: 0.95,
+ metadata: {},
+ } satisfies DocumentMemoryCard,
+ ],
+ );
+
+ const ranked = rankClinicalResults("ANC threshold stop clozapine", boosted);
+ expect(ranked[0].id).toBe("direct-threshold");
+ expect(ranked[0].memory_cards?.[0]?.id).toBe("card-1");
+ });
+
+ it("uses hybrid memory-card scores when mapping memory evidence back to chunks", () => {
+ const boosted = applyMemoryCardBoosts(
+ "table threshold monitoring",
+ [
+ result({
+ id: "table-threshold",
+ hybrid_score: 0.5,
+ content: "Table row: monitoring threshold requires escalation.",
+ }),
+ ],
+ [
+ {
+ id: "card-hybrid",
+ document_id: "doc-1",
+ owner_id: "user-1",
+ section_id: null,
+ card_type: "table_row",
+ title: "Monitoring threshold table row",
+ content: "Monitoring threshold requires escalation.",
+ normalized_terms: ["monitoring", "threshold"],
+ page_number: 1,
+ source_chunk_ids: ["table-threshold"],
+ source_image_ids: [],
+ confidence: 0.2,
+ metadata: { memory_hybrid_score: 0.9 },
+ } satisfies DocumentMemoryCard,
+ ],
+ );
+
+ expect(boosted[0].memory_score).toBe(0.9);
+ expect(boosted[0].hybrid_score).toBeGreaterThan(0.7);
+ });
+
+ it("persists memory cards without leaking internal section indexes into inserts", async () => {
+ const insertedMemoryRows: Record[] = [];
+ const supabase = {
+ from: vi.fn((table: string) => ({
+ delete: () => ({ eq: vi.fn(async () => ({ data: null, error: null })) }),
+ insert: (payload: Record[]) => {
+ if (table === "document_sections") {
+ return {
+ select: async () => ({
+ data: payload.map((section, index) => ({
+ id: `section-${index}`,
+ section_index: section.section_index,
+ })),
+ error: null,
+ }),
+ };
+ }
+ insertedMemoryRows.push(...payload);
+ return Promise.resolve({ data: null, error: null });
+ },
+ })),
+ rpc: vi.fn(async () => ({ data: null, error: null })),
+ };
+
+ await upsertDocumentDeepMemory({
+ supabase: supabase as never,
+ document,
+ chunks: [
+ chunk({
+ id: "chunk-upsert",
+ content: "If ANC is < 1.5, stop clozapine and urgent specialist review is required.",
+ }),
+ ],
+ });
+
+ expect(insertedMemoryRows.length).toBeGreaterThan(0);
+ expect(insertedMemoryRows.every((row) => !("section_index" in row))).toBe(true);
+ expect(insertedMemoryRows.every((row) => typeof row.section_id === "string" || row.section_id === null)).toBe(
+ true,
+ );
+ expect(supabase.rpc).toHaveBeenCalledWith("stamp_document_deep_memory_version", {
+ p_document_id: "doc-1",
+ p_version: ragDeepMemoryVersion,
+ });
+ });
+});
diff --git a/tests/document-enrichment.test.ts b/tests/document-enrichment.test.ts
new file mode 100644
index 000000000..2a7373390
--- /dev/null
+++ b/tests/document-enrichment.test.ts
@@ -0,0 +1,198 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ generateStructuredTextResponse: vi.fn(),
+}));
+
+vi.mock("@/lib/env", () => ({
+ env: {
+ OPENAI_FAST_ANSWER_MODEL: "gpt-fast-test",
+ },
+}));
+
+vi.mock("@/lib/openai", () => ({
+ generateStructuredTextResponse: mocks.generateStructuredTextResponse,
+}));
+
+import { generateDocumentEnrichment, ragEnrichmentVersion, upsertDocumentEnrichment } from "@/lib/document-enrichment";
+
+type QueryResult = { data: unknown; error: { message: string } | null };
+type QueryCall = {
+ table: string;
+ operation: "select" | "upsert" | "insert" | "update" | "delete";
+ payload?: unknown;
+ filters: Array<{ column: string; value: unknown }>;
+ selected?: string;
+};
+
+function createSupabaseMock() {
+ const calls: QueryCall[] = [];
+
+ class QueryBuilder implements PromiseLike {
+ constructor(private readonly call: QueryCall) {}
+
+ upsert(payload: unknown) {
+ this.call.operation = "upsert";
+ this.call.payload = payload;
+ return this;
+ }
+
+ insert(payload: unknown) {
+ this.call.operation = "insert";
+ this.call.payload = payload;
+ return this;
+ }
+
+ update(payload: unknown) {
+ this.call.operation = "update";
+ this.call.payload = payload;
+ return this;
+ }
+
+ delete() {
+ this.call.operation = "delete";
+ return this;
+ }
+
+ select(selected?: string) {
+ this.call.selected = selected;
+ return this;
+ }
+
+ eq(column: string, value: unknown) {
+ this.call.filters.push({ column, value });
+ return this;
+ }
+
+ single() {
+ return this.resolve();
+ }
+
+ then(
+ onfulfilled?: ((value: QueryResult) => TResult1 | PromiseLike) | null,
+ onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null,
+ ): PromiseLike {
+ return this.resolve().then(onfulfilled, onrejected);
+ }
+
+ private resolve() {
+ if (this.call.table === "document_summaries" && this.call.operation === "upsert") {
+ return Promise.resolve({ data: { id: "summary-1", ...(this.call.payload as object) }, error: null });
+ }
+ return Promise.resolve({ data: null, error: null });
+ }
+ }
+
+ const supabase = {
+ calls,
+ from: vi.fn((table: string) => {
+ const call: QueryCall = { table, operation: "select", filters: [] };
+ calls.push(call);
+ return new QueryBuilder(call);
+ }),
+ };
+
+ return supabase;
+}
+
+describe("document enrichment", () => {
+ beforeEach(() => {
+ mocks.generateStructuredTextResponse.mockResolvedValue(
+ JSON.stringify({
+ summary: "- Use the uploaded source for future-document clinical workflow review.",
+ clinical_specifics: {
+ actions: ["Check the source workflow."],
+ thresholds_timing: [],
+ medication_monitoring: [],
+ risk_escalation: [],
+ documentation_forms: [],
+ exceptions_gaps: [],
+ },
+ labels: [{ label: "future workflow", label_type: "workflow", confidence: 0.92 }],
+ }),
+ );
+ });
+
+ afterEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("writes the current RAG enrichment version to summaries, labels, and document metadata", async () => {
+ const supabase = createSupabaseMock();
+
+ await upsertDocumentEnrichment({
+ supabase: supabase as never,
+ document: {
+ id: "doc-future",
+ owner_id: null,
+ title: "Future Uploaded Protocol",
+ file_name: "future-upload.pdf",
+ source_path: null,
+ metadata: { existing: true },
+ },
+ chunks: [
+ {
+ id: "chunk-1",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: "Workflow",
+ content: "Future uploaded document workflow content for indexing and search.",
+ },
+ ],
+ images: [],
+ });
+
+ const summaryUpsert = supabase.calls.find((call) => call.table === "document_summaries" && call.operation === "upsert");
+ const labelsInsert = supabase.calls.find((call) => call.table === "document_labels" && call.operation === "insert");
+ const documentUpdate = supabase.calls.find((call) => call.table === "documents" && call.operation === "update");
+
+ expect((summaryUpsert?.payload as { metadata: Record }).metadata).toMatchObject({
+ generated_by: "local-worker",
+ rag_enrichment_version: ragEnrichmentVersion,
+ label_count: expect.any(Number),
+ coverage_profile: expect.objectContaining({ chunk_count: 1 }),
+ });
+ expect(labelsInsert?.payload).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ document_id: "doc-future",
+ source: "generated",
+ metadata: expect.objectContaining({ rag_enrichment_version: ragEnrichmentVersion }),
+ }),
+ ]),
+ );
+ expect((documentUpdate?.payload as { metadata: Record }).metadata).toMatchObject({
+ existing: true,
+ rag_enrichment_version: ragEnrichmentVersion,
+ generated_label_count: expect.any(Number),
+ coverage_profile: expect.objectContaining({ chunk_count: 1 }),
+ });
+ expect(documentUpdate?.filters).toContainEqual({ column: "id", value: "doc-future" });
+ });
+
+ it("uses coverage-aware source excerpts instead of only the first chunks for large documents", async () => {
+ await generateDocumentEnrichment({
+ document: {
+ title: "Large Clozapine Protocol",
+ file_name: "large-clozapine.pdf",
+ source_path: null,
+ },
+ chunks: Array.from({ length: 60 }, (_, index) => ({
+ id: `chunk-${index}`,
+ page_number: index + 1,
+ chunk_index: index,
+ section_heading: index % 10 === 0 ? `Section ${index}` : null,
+ content:
+ index === 52
+ ? "If ANC is < 1.5, stop clozapine and seek urgent specialist review."
+ : `Routine source content ${index}.`,
+ })),
+ images: [],
+ });
+
+ const prompt = String(mocks.generateStructuredTextResponse.mock.calls.at(-1)?.[0] ?? "");
+ expect(prompt).toContain("Coverage: 60 indexed chunks");
+ expect(prompt).toContain("chunk_id: chunk-52");
+ expect(prompt).toContain("remain indexed and retrievable");
+ });
+});
diff --git a/tests/document-naming.test.ts b/tests/document-naming.test.ts
new file mode 100644
index 000000000..dfce1d51a
--- /dev/null
+++ b/tests/document-naming.test.ts
@@ -0,0 +1,77 @@
+import { describe, expect, it } from "vitest";
+import { documentTitleKey, planDocumentName, smartDocumentTitle } from "../src/lib/document-naming";
+
+function supabaseWithDocuments(documents: unknown[]) {
+ return {
+ from: () => ({
+ select: () => ({
+ eq: () => ({
+ limit: async () => ({ data: documents, error: null }),
+ }),
+ }),
+ }),
+ };
+}
+
+describe("document naming", () => {
+ it("creates readable titles from compact clinical filenames", () => {
+ expect(smartDocumentTitle("MHSP.AgitationArousalPharmaMgt.pdf")).toBe(
+ "MHSP - Agitation Arousal Pharmacological Management",
+ );
+ expect(smartDocumentTitle("clozapine_pres_admin_monitor_v5.0.pdf")).toBe(
+ "Clozapine Prescribing Administering Monitoring V5.0",
+ );
+ });
+
+ it("uses stable duplicate keys for equivalent names", () => {
+ expect(documentTitleKey("Guideline (Copy 2)")).toBe(documentTitleKey("Guideline"));
+ });
+
+ it("keeps first upload title clean when no same-name document exists", async () => {
+ const plan = await planDocumentName({
+ supabase: supabaseWithDocuments([]),
+ ownerId: "owner",
+ fileName: "guideline.pdf",
+ contentHash: "hash-1",
+ });
+
+ expect(plan).toMatchObject({
+ title: "Guideline",
+ baseTitle: "Guideline",
+ duplicateIndex: 1,
+ duplicateReason: "none",
+ });
+ });
+
+ it("adds a clear suffix when a different document has the same title or filename", async () => {
+ const plan = await planDocumentName({
+ supabase: supabaseWithDocuments([
+ { id: "doc-1", title: "Guideline", file_name: "guideline.pdf", content_hash: "hash-1" },
+ ]),
+ ownerId: "owner",
+ fileName: "guideline.pdf",
+ contentHash: "hash-2",
+ });
+
+ expect(plan).toMatchObject({
+ title: "Guideline (Copy 2)",
+ baseTitle: "Guideline",
+ duplicateIndex: 2,
+ duplicateReason: "same_title_or_filename",
+ });
+ });
+
+ it("prefers a version/date suffix from the uploaded filename when available", async () => {
+ const plan = await planDocumentName({
+ supabase: supabaseWithDocuments([
+ { id: "doc-1", title: "Clozapine Prescribing", file_name: "clozapine_prescribing.pdf", content_hash: "hash-1" },
+ ]),
+ ownerId: "owner",
+ fileName: "clozapine_prescribing_v5.0.pdf",
+ requestedTitle: "Clozapine Prescribing",
+ contentHash: "hash-2",
+ });
+
+ expect(plan.title).toBe("Clozapine Prescribing (v5.0)");
+ });
+});
diff --git a/tests/eval-search.test.ts b/tests/eval-search.test.ts
new file mode 100644
index 000000000..2763f427a
--- /dev/null
+++ b/tests/eval-search.test.ts
@@ -0,0 +1,71 @@
+import { describe, expect, it } from "vitest";
+import { expectedFileCoverage, expectedFileHit } from "../scripts/eval-utils";
+import { summarizeFailures, type SearchEvalResult } from "../scripts/eval-search";
+
+function result(overrides: Partial = {}): SearchEvalResult {
+ return {
+ id: "custom-question",
+ question: "agitation and arousal",
+ category: "routine",
+ supported: true,
+ expectedFileCount: 1,
+ expectedHitTop3: true,
+ expectedAllHitTop5: null,
+ missingExpectedFiles: [],
+ resultCount: 1,
+ topScore: 0.97,
+ topFiles: ["MHSP.AgitationArousalPharmaMgt.pdf"],
+ latencyMs: 300,
+ retrievalStrategy: "text_fast_path",
+ searchCacheHit: false,
+ embeddingSkipped: true,
+ embeddingCacheHit: false,
+ fallbackToEmbedding: false,
+ visualEvidence: 0,
+ failures: [],
+ ...overrides,
+ };
+}
+
+describe("search eval thresholds", () => {
+ it("does not count unsupported cases with no expected files as expected hits", () => {
+ expect(expectedFileHit([], [{ file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf" }])).toBe(false);
+ expect(expectedFileCoverage([], [{ file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf" }]).allHit).toBe(false);
+ });
+
+ it("requires all expected files for multi-document coverage", () => {
+ const partial = expectedFileCoverage(
+ ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+ [{ file_name: "MHSP.Discharge.pdf" }],
+ 5,
+ );
+
+ expect(partial.anyHit).toBe(true);
+ expect(partial.allHit).toBe(false);
+ expect(partial.missingFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf"]);
+ });
+
+ it("does not apply full-suite aggregate hit thresholds to a targeted question run", () => {
+ expect(summarizeFailures([result()])).toEqual([]);
+ });
+
+ it("still reports case-level failures for targeted question runs", () => {
+ expect(summarizeFailures([result({ failures: ["expected document not in top 3"] })])).toContain(
+ "supported case-level search failure(s)",
+ );
+ });
+
+ it("reports multi-document case-level failures when only one expected file is present", () => {
+ expect(
+ summarizeFailures([
+ result({
+ expectedFileCount: 2,
+ expectedHitTop3: true,
+ expectedAllHitTop5: false,
+ missingExpectedFiles: ["MHSP.AdmissionCommunityPts.pdf"],
+ failures: ["expected documents missing from top 5: MHSP.AdmissionCommunityPts.pdf"],
+ }),
+ ]),
+ ).toContain("supported case-level search failure(s)");
+ });
+});
diff --git a/tests/evidence-relevance.test.ts b/tests/evidence-relevance.test.ts
new file mode 100644
index 000000000..921f9ace9
--- /dev/null
+++ b/tests/evidence-relevance.test.ts
@@ -0,0 +1,132 @@
+import { describe, expect, it } from "vitest";
+import { buildVisualEvidence } from "../src/lib/evidence";
+import {
+ annotateSearchResults,
+ buildEvidenceRelevance,
+ buildSourceRelevance,
+} from "../src/lib/evidence-relevance";
+import type { SearchResult } from "../src/lib/types";
+
+function result(overrides: Partial = {}): SearchResult {
+ return {
+ id: "chunk-1",
+ document_id: "doc-1",
+ title: "Clinical source",
+ file_name: "source.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: "Monitoring",
+ content: "Clozapine monitoring requires FBC, ANC, myocarditis review, metabolic review, and constipation planning.",
+ image_ids: [],
+ similarity: 0.88,
+ hybrid_score: 0.88,
+ source_strength: "strong",
+ images: [],
+ ...overrides,
+ };
+}
+
+describe("evidence relevance", () => {
+ it("classifies weak lithium-style neighboring sources as nearby, not direct", () => {
+ const relevance = buildEvidenceRelevance("What toxicity safety-net symptoms should be reviewed for lithium?", [
+ result({
+ id: "clozapine-neighbor",
+ title: "Clozapine monitoring",
+ content: "Monitor for toxicity symptoms, red flags, and urgent review during clozapine treatment.",
+ similarity: 0.37,
+ hybrid_score: 0.37,
+ text_rank: 0,
+ source_strength: "limited",
+ }),
+ ]);
+
+ expect(relevance.verdict).toBe("nearby");
+ expect(relevance.isSourceBacked).toBe(false);
+ expect(relevance.missingTerms).toContain("lithium");
+ });
+
+ it("classifies strong direct concept coverage as direct", () => {
+ const relevance = buildEvidenceRelevance("clozapine monitoring", [result()]);
+
+ expect(relevance.verdict).toBe("direct");
+ expect(relevance.isSourceBacked).toBe(true);
+ expect(relevance.directSourceCount).toBe(1);
+ });
+
+ it("exposes missing terms for partial support", () => {
+ const relevance = buildEvidenceRelevance("lithium toxicity vomiting dehydration advice", [
+ result({
+ title: "Lithium toxicity",
+ content: "Lithium toxicity symptoms include vomiting and dehydration.",
+ similarity: 0.86,
+ hybrid_score: 0.86,
+ }),
+ ]);
+
+ expect(relevance.verdict).toBe("partial");
+ expect(relevance.isSourceBacked).toBe(true);
+ expect(relevance.missingTerms).toContain("advice");
+ });
+
+ it("adds concise query coverage chips to source results", () => {
+ const [source] = annotateSearchResults("lithium toxicity advice", [
+ result({
+ title: "Lithium toxicity",
+ content: "Lithium toxicity symptoms are described.",
+ }),
+ ]);
+
+ expect(source.relevance?.chips.join(" ")).toContain("matched:");
+ expect(source.relevance?.chips.join(" ")).toContain("missing:");
+ });
+
+ it("prioritizes direct or partial visual evidence above nearby-only images", () => {
+ const direct = result({
+ id: "direct",
+ document_id: "doc-direct",
+ title: "Clozapine monitoring",
+ content: "Clozapine monitoring table.",
+ relevance: buildSourceRelevance("clozapine monitoring", result({ title: "Clozapine monitoring" })),
+ images: [
+ {
+ id: "img-direct",
+ page_number: 1,
+ storage_path: "private/direct.png",
+ caption: "Direct clozapine monitoring table.",
+ searchable: true,
+ image_type: "clinical_table",
+ source_kind: "table_crop",
+ tableRole: "clinical",
+ clinicalUseClass: "clinical_evidence",
+ clinical_relevance_score: 0.1,
+ },
+ ],
+ });
+ const nearby = result({
+ id: "nearby",
+ document_id: "doc-nearby",
+ title: "Generic safety",
+ content: "Safety monitoring table.",
+ relevance: buildSourceRelevance("clozapine monitoring", result({ title: "Generic safety", content: "Safety monitoring table." })),
+ images: [
+ {
+ id: "img-nearby",
+ page_number: 1,
+ storage_path: "private/nearby.png",
+ caption: "Nearby safety monitoring table.",
+ searchable: true,
+ image_type: "clinical_table",
+ source_kind: "table_crop",
+ tableRole: "clinical",
+ clinicalUseClass: "clinical_evidence",
+ clinical_relevance_score: 1,
+ },
+ ],
+ });
+
+ const cards = buildVisualEvidence([nearby, direct], 2);
+
+ expect(cards[0].image_id).toBe("img-direct");
+ expect(cards[1].relevance?.isSourceBacked).toBe(false);
+ });
+});
diff --git a/tests/evidence-tags.test.ts b/tests/evidence-tags.test.ts
new file mode 100644
index 000000000..72a1a9246
--- /dev/null
+++ b/tests/evidence-tags.test.ts
@@ -0,0 +1,24 @@
+import { describe, expect, it } from "vitest";
+import { smartEvidenceTags } from "../src/lib/evidence-tags";
+
+describe("smart evidence tags", () => {
+ it("turns generic role labels into clinically useful tags", () => {
+ const tags = smartEvidenceTags(
+ ["roles", "responsibilities", "clozapine monitoring", "psychiatrist", "care coordinator"],
+ "Roles And Responsibilities table for Clozapine monitoring, psychiatrist oversight, and care coordination.",
+ );
+
+ expect(tags).toEqual([
+ "Clozapine monitoring",
+ "Psychiatrist review",
+ "Care team responsibilities",
+ "Care coordination",
+ ]);
+ });
+
+ it("deduplicates and capitalizes raw generated labels", () => {
+ const tags = smartEvidenceTags(["blood_tests", "blood test", "dose", "monitoring"], "Clozapine dose table");
+
+ expect(tags).toEqual(["Blood test monitoring", "Dose adjustment", "Clozapine monitoring"]);
+ });
+});
diff --git a/tests/evidence.test.ts b/tests/evidence.test.ts
index c3d06d088..4335e081e 100644
--- a/tests/evidence.test.ts
+++ b/tests/evidence.test.ts
@@ -37,6 +37,32 @@ describe("evidence helpers", () => {
expect(quotes[0].page_number).toBe(1);
});
+ it("selects direct cease or discontinue wording for clozapine withhold threshold questions", () => {
+ const quotes = extractQuoteCards(
+ [
+ result({
+ id: "clozapine-threshold",
+ title: "Clozapine Prescribing",
+ content: `Clozapine Prescribing, Administering and Monitoring
+
+State WBC Neutrophil Outcome
+Green >= 3.5 >= 2 Continue with regular blood tests
+Amber >=3.0 and <3.5 >= 1.5 and <2.0 Twice weekly blood tests required
+Red <3 < 1.5 Cease therapy immediately
+
+If the consumer's blood results return in the red range, Clozapine therapy must be discontinued immediately.
+The haematologist can assist with altering WCC and ANC thresholds for specific consumers.`,
+ }),
+ ],
+ "What FBC threshold should withhold clozapine?",
+ );
+
+ expect(quotes[0].quote.toLowerCase()).toMatch(/cease|discontinued/);
+ expect(quotes[0].quote).toContain("State WBC Neutrophil Outcome");
+ expect(quotes[0].quote).toContain("Red <3 < 1.5 Cease therapy immediately");
+ expect(quotes[0].quote.toLowerCase()).not.toContain("haematologist can assist");
+ });
+
it("uses image descriptions as quotable indexed evidence", () => {
const quotes = extractQuoteCards(
[
@@ -88,6 +114,18 @@ describe("evidence helpers", () => {
expect(diversified.filter((source) => source.document_id === "doc-a")).toHaveLength(2);
});
+ it("can preserve upstream clinical ranking while capping document dominance", () => {
+ const sources = [
+ result({ id: "ranked-lower-hybrid", document_id: "doc-a", hybrid_score: 0.6, similarity: 0.6 }),
+ result({ id: "ranked-higher-hybrid", document_id: "doc-b", hybrid_score: 0.9, similarity: 0.9 }),
+ ];
+
+ expect(diversifySearchResults(sources, 2, 4, true).map((source) => source.id)).toEqual([
+ "ranked-lower-hybrid",
+ "ranked-higher-hybrid",
+ ]);
+ });
+
it("falls back to locally extracted exact quotes when proposed quotes are not exact", () => {
const sources = [result({ id: "a1" })];
const quotes = reconcileQuoteCards(
@@ -179,6 +217,20 @@ describe("evidence helpers", () => {
page_number: 3,
storage_path: "private/path/image.png",
caption: "A source diagram extracted from the indexed PDF.",
+ searchable: true,
+ image_type: "clinical_table",
+ source_kind: "table_crop",
+ tableLabel: "Table 1",
+ tableTitle: "Agitation and arousal rating scale",
+ tableRole: "clinical",
+ clinicalUseClass: "clinical_evidence",
+ accessibleTableMarkdown: "| Score | Management |\n| --- | --- |\n| 0 | Monitor observations |",
+ tableRows: [
+ ["Score", "Management"],
+ ["0", "Monitor observations"],
+ ],
+ tableColumns: ["Score", "Management"],
+ tableTextSnippet: "Score 0 | Asleep or unconscious",
},
],
}),
@@ -189,6 +241,41 @@ describe("evidence helpers", () => {
expect(cards[0].image_id).toBe("img-1");
expect(cards[0].signed_url_endpoint).toBe("/api/images/img-1/signed-url");
expect(cards[0].viewer_href).toContain("page=3");
+ expect(cards[0].tableLabel).toBe("Table 1");
+ expect(cards[0].tableTitle).toContain("Agitation");
+ expect(cards[0].tableRole).toBe("clinical");
+ expect(cards[0].clinicalUseClass).toBe("clinical_evidence");
+ expect(cards[0].accessibleTableMarkdown).toContain("Score");
+ expect(cards[0].tableRows?.[1]?.[1]).toContain("Monitor");
expect(cards[0]).not.toHaveProperty("storage_path");
});
+
+ it("excludes administrative tables from visual evidence cards", () => {
+ const cards = buildVisualEvidence([
+ result({
+ id: "admin-image-source",
+ images: [
+ {
+ id: "img-admin",
+ page_number: 3,
+ storage_path: "private/path/admin.png",
+ caption: "Authorisation and publication table.",
+ searchable: true,
+ image_type: "clinical_table",
+ source_kind: "table_crop",
+ tableRole: "admin",
+ clinicalUseClass: "administrative",
+ tableTextSnippet: "Authorised by | Authorisation date | Published date",
+ metadata: {
+ clinical_use_class: "administrative",
+ table_role: "admin",
+ table_text: "Authorised by | Authorisation date | Published date",
+ },
+ },
+ ],
+ }),
+ ]);
+
+ expect(cards).toEqual([]);
+ });
});
diff --git a/tests/image-filtering.test.ts b/tests/image-filtering.test.ts
index 20f209448..0ec9873f2 100644
--- a/tests/image-filtering.test.ts
+++ b/tests/image-filtering.test.ts
@@ -1,7 +1,9 @@
import { describe, expect, it } from "vitest";
import {
+ assessClinicalImageUse,
cheapImageSkipReason,
classifiedImageSkipReason,
+ isClinicalImageEvidence,
lightweightPerceptualHash,
} from "../src/lib/image-filtering";
@@ -29,6 +31,17 @@ describe("smart image filtering", () => {
).toBe("logo/header/footer placement");
});
+ it("does not skip table crops because they sit near a page header", () => {
+ expect(
+ cheapImageSkipReason({
+ bytesLength: 20_000,
+ imageHash: "table",
+ seenHashes: new Set(),
+ image: { sourceKind: "table_crop", width: 720, height: 180, bbox: [20, 20, 740, 200] },
+ }),
+ ).toBeNull();
+ });
+
it("keeps relevant clinical classifications searchable", () => {
expect(
classifiedImageSkipReason({
@@ -40,6 +53,79 @@ describe("smart image filtering", () => {
).toBeNull();
});
+ it("classifies authorisation and publication tables as administrative evidence", () => {
+ const assessment = assessClinicalImageUse({
+ imageType: "clinical_table",
+ searchable: true,
+ clinicalRelevanceScore: 0.8,
+ sourceKind: "table_crop",
+ tableText:
+ "| Authorised by | Karen Elliott |\n| Authorisation date | 4/11/2024 |\n| Published date | 13/11/2024 |",
+ });
+
+ expect(assessment.clinical_use_class).toBe("administrative");
+ expect(assessment.searchable).toBe(false);
+ expect(assessment.clinical_relevance_score).toBe(0);
+ });
+
+ it("classifies version and amendment tables as administrative evidence", () => {
+ expect(
+ assessClinicalImageUse({
+ imageType: "clinical_table",
+ searchable: true,
+ clinicalRelevanceScore: 0.7,
+ sourceKind: "table_crop",
+ tableText: "| Version | Effective from | Effective to | Amendment(s) |\n| V5.0 | 13/11/2024 | 13/11/2027 | Link added |",
+ }).clinical_use_class,
+ ).toBe("administrative");
+ });
+
+ it("keeps medication, monitoring, threshold, and workflow tables clinical", () => {
+ const assessment = assessClinicalImageUse({
+ imageType: "clinical_table",
+ searchable: true,
+ clinicalRelevanceScore: 0.6,
+ sourceKind: "table_crop",
+ tableText:
+ "| Score | Patient state | Management |\n| 5 | Highly aroused | Oral lorazepam 1 mg and monitor observations for escalation risk |",
+ });
+
+ expect(assessment.clinical_use_class).toBe("clinical_evidence");
+ expect(assessment.searchable).toBe(true);
+ });
+
+ it("treats role tables as clinical only when responsibilities affect patient care", () => {
+ expect(
+ assessClinicalImageUse({
+ imageType: "clinical_table",
+ sourceKind: "table_crop",
+ tableText: "| Role | Responsibility |\n| Service Director | Overall responsibility for policy governance and compliance |",
+ }).clinical_use_class,
+ ).not.toBe("clinical_evidence");
+
+ expect(
+ assessClinicalImageUse({
+ imageType: "clinical_table",
+ sourceKind: "table_crop",
+ tableText:
+ "| Role | Responsibility |\n| Clozapine nurse | Monitor patient observations and escalate abnormal blood results |",
+ }).clinical_use_class,
+ ).toBe("clinical_evidence");
+ });
+
+ it("does not treat site and applicability cover tables as clinical evidence", () => {
+ expect(
+ isClinicalImageEvidence({
+ image_type: "clinical_table",
+ searchable: true,
+ source_kind: "table_crop",
+ metadata: {
+ table_text: "| Site | Operational Area | Applicable to |\n| Armadale | Mental Health | Medical staff |",
+ },
+ }),
+ ).toBe(false);
+ });
+
it("skips decorative classifications", () => {
expect(
classifiedImageSkipReason({
diff --git a/tests/indexed-source-formatting.test.ts b/tests/indexed-source-formatting.test.ts
new file mode 100644
index 000000000..f06931d26
--- /dev/null
+++ b/tests/indexed-source-formatting.test.ts
@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import { parseIndexedSourceText } from "../src/lib/indexed-source-formatting";
+
+describe("indexed source formatting", () => {
+ it("turns raw PDF page extraction into headings, paragraphs, lists, and tables", () => {
+ const blocks = parseIndexedSourceText(`
+ Clozapine Prescribing, Administering and Monitoring
+
+NB Clinical judgement and or emerging signs and symptoms will determine appropriate
+intervals of monitoring outside the recommended parameters.
+
+9. Polypharmacy
+Additionally, consumers may be taking other medications which require monitoring including:
+• Amisulpride, Risperidone, Olanzapine > 20mg, Paliperidone - prolactin
+• Quetiapine - TSH
+
+11. Therapy Interruption
+The Clozapine monitoring protocol must be followed if a patient's blood test is missed.
+
+ Time since last Clozapine Clozapine dose Blood test monitoring
+ dose
+
+ <48 hours Restart at normal dose of No changes to monitoring
+ Clozapine
+
+>= 48 hours - <=72 hours Restart Clozapine at 12.5mg. No changes to monitoring
+ Consideration for inpatient
+ admission.
+
+ >=72 hours- <4 weeks Restart Clozapine at 12.5mg Patient on weekly monitoring:
+ and retitrate in IPU Continue weekly blood tests
+ for at least 6 weeks
+
+ Page 8 of 15
+`);
+
+ expect(blocks[0]).toMatchObject({
+ type: "heading",
+ level: "title",
+ text: "Clozapine Prescribing, Administering and Monitoring",
+ });
+ expect(blocks).toContainEqual(expect.objectContaining({ type: "heading", text: "9. Polypharmacy" }));
+ expect(blocks).toContainEqual(
+ expect.objectContaining({
+ type: "paragraph",
+ text: "NB Clinical judgement and or emerging signs and symptoms will determine appropriate intervals of monitoring outside the recommended parameters.",
+ }),
+ );
+ expect(blocks).toContainEqual(
+ expect.objectContaining({
+ type: "list",
+ items: [
+ "Amisulpride, Risperidone, Olanzapine > 20mg, Paliperidone - prolactin",
+ "Quetiapine - TSH",
+ ],
+ }),
+ );
+
+ const table = blocks.find((block) => block.type === "table");
+ expect(table).toMatchObject({
+ type: "table",
+ rows: [
+ ["Time since last Clozapine dose", "Clozapine dose", "Blood test monitoring"],
+ ["<48 hours", "Restart at normal dose of Clozapine", "No changes to monitoring"],
+ [
+ ">= 48 hours - <=72 hours",
+ "Restart Clozapine at 12.5mg. Consideration for inpatient admission.",
+ "No changes to monitoring",
+ ],
+ [
+ ">=72 hours- <4 weeks",
+ "Restart Clozapine at 12.5mg and retitrate in IPU",
+ "Patient on weekly monitoring: Continue weekly blood tests for at least 6 weeks",
+ ],
+ ],
+ });
+ expect(JSON.stringify(blocks)).not.toContain("Page 8 of 15");
+ });
+});
diff --git a/tests/indexing-coverage.test.ts b/tests/indexing-coverage.test.ts
new file mode 100644
index 000000000..064ec4fef
--- /dev/null
+++ b/tests/indexing-coverage.test.ts
@@ -0,0 +1,66 @@
+import { describe, expect, it } from "vitest";
+import {
+ buildCoveragePromptNote,
+ buildIndexingCoverageProfile,
+ selectCoverageAwarePromptChunks,
+} from "../src/lib/indexing-coverage";
+
+function chunk(index: number, content = `Routine page content ${index}.`) {
+ return {
+ id: `chunk-${index}`,
+ page_number: index,
+ chunk_index: index,
+ section_heading: index % 5 === 0 ? `Section ${index}` : null,
+ content,
+ };
+}
+
+describe("indexing coverage", () => {
+ it("records complete page and chunk coverage without hiding missing pages", () => {
+ const profile = buildIndexingCoverageProfile({
+ pageCount: 4,
+ chunks: [chunk(1), chunk(2), chunk(4)],
+ images: [{ id: "image-1", page_number: 4, caption: "Clinical table" }],
+ });
+
+ expect(profile).toMatchObject({
+ chunk_count: 3,
+ image_count: 1,
+ page_coverage_count: 3,
+ expected_page_count: 4,
+ missing_page_numbers: [3],
+ has_complete_page_chunk_coverage: false,
+ });
+ });
+
+ it("selects coverage-aware enrichment chunks across large documents", () => {
+ const chunks = Array.from({ length: 80 }, (_, index) =>
+ chunk(
+ index,
+ index === 60
+ ? "If ANC is < 1.5, stop clozapine and seek urgent specialist review."
+ : `Routine source content ${index}.`,
+ ),
+ );
+
+ const selected = selectCoverageAwarePromptChunks(chunks, 12);
+ const ids = selected.chunks.map((item) => item.id);
+
+ expect(selected.strategy).toBe("coverage_spread_high_yield_headings");
+ expect(ids).toContain("chunk-0");
+ expect(ids).toContain("chunk-79");
+ expect(ids).toContain("chunk-60");
+ expect(selected.chunks.length).toBeLessThanOrEqual(12);
+ });
+
+ it("makes prompt truncation explicit as excerpt selection, not lost indexing", () => {
+ const profile = buildIndexingCoverageProfile({ pageCount: 20, chunks: Array.from({ length: 20 }, (_, index) => chunk(index + 1)) });
+ const note = buildCoveragePromptNote({
+ profile,
+ selectedChunkIds: ["chunk-1", "chunk-10", "chunk-20"],
+ });
+
+ expect(note).toContain("20 indexed chunks");
+ expect(note).toContain("17 chunk(s) remain indexed and retrievable");
+ });
+});
diff --git a/tests/openai-cache.test.ts b/tests/openai-cache.test.ts
index f87c07f8d..67485b24c 100644
--- a/tests/openai-cache.test.ts
+++ b/tests/openai-cache.test.ts
@@ -114,12 +114,16 @@ describe("OpenAI query embedding cache", () => {
}));
const { generateStructuredTextResult } = await import("../src/lib/openai");
- const result = await generateStructuredTextResult("Question", { type: "object", properties: {}, required: [] }, {
- model: "gpt-5.4-mini",
- operation: "answer",
- schemaName: "clinical_test",
- maxOutputTokens: 200,
- });
+ const result = await generateStructuredTextResult(
+ "Question",
+ { type: "object", properties: {}, required: [] },
+ {
+ model: "gpt-5.4-mini",
+ operation: "answer",
+ schemaName: "clinical_test",
+ maxOutputTokens: 200,
+ },
+ );
expect(result.requestId).toBe("req_123");
expect(result.usage).toMatchObject({
diff --git a/tests/pdf-extractor.test.ts b/tests/pdf-extractor.test.ts
new file mode 100644
index 000000000..2bfab6efe
--- /dev/null
+++ b/tests/pdf-extractor.test.ts
@@ -0,0 +1,142 @@
+import { spawnSync } from "node:child_process";
+import { createWriteStream } from "node:fs";
+import { mkdir, mkdtemp, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import PDFDocument from "pdfkit";
+import { describe, expect, it } from "vitest";
+
+const pythonBin = process.env.PYTHON_BIN || "python";
+const hasPyMuPDF = spawnSync(pythonBin, ["-c", "import fitz"], { encoding: "utf8" }).status === 0;
+
+async function writeSyntheticTablePdf(filePath: string) {
+ await new Promise((resolve, reject) => {
+ const doc = new PDFDocument({ size: "A4", margin: 40 });
+ const stream = createWriteStream(filePath);
+ stream.on("finish", resolve);
+ stream.on("error", reject);
+ doc.pipe(stream);
+
+ doc.fontSize(12).text("Table 1: Agitation and arousal rating scale and associated management", 50, 64);
+ const x = 50;
+ const y = 92;
+ const widths = [70, 190, 250];
+ const rowHeight = 34;
+ const rows = [
+ ["Score", "Patient's state", "Management"],
+ ["0", "Asleep or unconscious", "Review prescribed doses and conduct required monitoring."],
+ ["5", "Highly aroused and violent", "Use emergency escalation and record observations."],
+ ];
+
+ let currentY = y;
+ for (const row of rows) {
+ let currentX = x;
+ for (let index = 0; index < row.length; index += 1) {
+ doc.rect(currentX, currentY, widths[index], rowHeight).stroke();
+ doc.fontSize(index === 0 ? 8 : 7).text(row[index], currentX + 4, currentY + 6, {
+ width: widths[index] - 8,
+ height: rowHeight - 8,
+ });
+ currentX += widths[index];
+ }
+ currentY += rowHeight;
+ }
+
+ doc.end();
+ });
+}
+
+async function writeSyntheticAdminTablePdf(filePath: string) {
+ await new Promise((resolve, reject) => {
+ const doc = new PDFDocument({ size: "A4", margin: 40 });
+ const stream = createWriteStream(filePath);
+ stream.on("finish", resolve);
+ stream.on("error", reject);
+ doc.pipe(stream);
+
+ doc.fontSize(12).text("13. Authorisation", 50, 64);
+ const x = 50;
+ const y = 92;
+ const widths = [160, 340];
+ const rowHeight = 30;
+ const rows = [
+ ["Authorisation date", "Published date"],
+ ["01/01/2026", "02/01/2026"],
+ ["Document owner", "Mental Health Service"],
+ ];
+
+ let currentY = y;
+ for (const row of rows) {
+ let currentX = x;
+ for (let index = 0; index < row.length; index += 1) {
+ doc.rect(currentX, currentY, widths[index], rowHeight).stroke();
+ doc.fontSize(8).text(row[index], currentX + 4, currentY + 7, {
+ width: widths[index] - 8,
+ height: rowHeight - 8,
+ });
+ currentX += widths[index];
+ }
+ currentY += rowHeight;
+ }
+
+ doc.end();
+ });
+}
+
+describe.runIf(hasPyMuPDF)("Python PDF table extraction", () => {
+ it("writes clean JSON to a file and emits clinical table crops with titles", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "clinical-kb-extractor-test-"));
+ const pdfPath = path.join(root, "table.pdf");
+ const imageDir = path.join(root, "images");
+ const jsonPath = path.join(root, "extract.json");
+ await mkdir(imageDir, { recursive: true });
+ await writeSyntheticTablePdf(pdfPath);
+
+ const result = spawnSync(
+ pythonBin,
+ [path.join(process.cwd(), "worker", "python", "extract_pdf_assets.py"), pdfPath, imageDir, jsonPath],
+ { cwd: process.cwd(), encoding: "utf8" },
+ );
+
+ expect(result.status).toBe(0);
+ const payload = JSON.parse(await readFile(jsonPath, "utf8")) as {
+ images: Array<{ sourceKind?: string; metadata?: Record }>;
+ };
+ const tableCrop = payload.images.find((image) => image.sourceKind === "table_crop");
+ expect(tableCrop).toBeTruthy();
+ expect(tableCrop?.metadata?.table_title).toContain("Agitation and arousal");
+ expect(tableCrop?.metadata?.table_text).toContain("Management");
+ expect(tableCrop?.metadata?.table_role).toBe("clinical");
+ expect(Number(tableCrop?.metadata?.table_confidence)).toBeGreaterThan(0.5);
+ expect(tableCrop?.metadata?.accessible_table_markdown).toContain("Score");
+ expect(tableCrop?.metadata?.table_columns).toEqual(["Score", "Patient's state", "Management"]);
+ expect(tableCrop?.metadata?.table_rows).toEqual(
+ expect.arrayContaining([expect.arrayContaining(["5", "Highly aroused and violent"])]),
+ );
+ });
+
+ it("retains real administrative tables with a non-clinical role for document review", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "clinical-kb-extractor-test-"));
+ const pdfPath = path.join(root, "admin-table.pdf");
+ const imageDir = path.join(root, "images");
+ const jsonPath = path.join(root, "extract.json");
+ await mkdir(imageDir, { recursive: true });
+ await writeSyntheticAdminTablePdf(pdfPath);
+
+ const result = spawnSync(
+ pythonBin,
+ [path.join(process.cwd(), "worker", "python", "extract_pdf_assets.py"), pdfPath, imageDir, jsonPath],
+ { cwd: process.cwd(), encoding: "utf8" },
+ );
+
+ expect(result.status).toBe(0);
+ const payload = JSON.parse(await readFile(jsonPath, "utf8")) as {
+ images: Array<{ sourceKind?: string; metadata?: Record }>;
+ };
+ const tableCrop = payload.images.find((image) => image.sourceKind === "table_crop");
+ expect(tableCrop).toBeTruthy();
+ expect(tableCrop?.metadata?.table_text).toContain("Authorisation date");
+ expect(tableCrop?.metadata?.table_role).toBe("admin");
+ expect(tableCrop?.metadata?.accessible_table_markdown).toContain("Published date");
+ });
+});
diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts
index ff0b81115..7a0e4be72 100644
--- a/tests/private-access-routes.test.ts
+++ b/tests/private-access-routes.test.ts
@@ -13,11 +13,15 @@ type QueryFilter = { column: string; value: unknown };
type QueryInFilter = { column: string; values: unknown[] };
type QueryCall = {
table: string;
- operation: "select" | "insert";
+ operation: "select" | "insert" | "update" | "delete";
selected?: string;
+ range?: { from: number; to: number };
+ orFilters: string[];
filters: QueryFilter[];
inFilters: QueryInFilter[];
+ overlapsFilters: QueryInFilter[];
insertPayload?: unknown;
+ updatePayload?: unknown;
limitCount?: number;
maybeSingle: boolean;
single: boolean;
@@ -49,20 +53,56 @@ class QueryBuilder implements PromiseLike {
return this;
}
+ update(payload: unknown) {
+ this.call.operation = "update";
+ this.call.updatePayload = payload;
+ return this;
+ }
+
+ delete() {
+ this.call.operation = "delete";
+ return this;
+ }
+
eq(column: string, value: unknown) {
this.call.filters.push({ column, value });
return this;
}
+ neq(column: string, value: unknown) {
+ this.call.filters.push({ column, value });
+ return this;
+ }
+
+ not(column: string, _operator: string, value: unknown) {
+ this.call.filters.push({ column, value });
+ return this;
+ }
+
in(column: string, values: unknown[]) {
this.call.inFilters.push({ column, values });
return this;
}
+ overlaps(column: string, values: unknown[]) {
+ this.call.overlapsFilters.push({ column, values });
+ return this;
+ }
+
order() {
return this;
}
+ range(from: number, to: number) {
+ this.call.range = { from, to };
+ return this;
+ }
+
+ or(filter: string) {
+ this.call.orFilters.push(filter);
+ return this;
+ }
+
limit(count: number) {
this.call.limitCount = count;
return this;
@@ -121,8 +161,10 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) {
const call: QueryCall = {
table,
operation: "select",
+ orFilters: [],
filters: [],
inFilters: [],
+ overlapsFilters: [],
maybeSingle: false,
single: false,
};
@@ -137,10 +179,16 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) {
return client;
}
-function mockRuntime(client: ReturnType, ragMock?: Record) {
+function mockRuntime(
+ client: ReturnType,
+ ragMock?: Record,
+ options: { localNoAuth?: boolean } = {},
+) {
vi.resetModules();
vi.doUnmock("@/lib/rag");
vi.doUnmock("@/lib/openai");
+ vi.doUnmock("@/lib/document-enrichment");
+ vi.doUnmock("@/lib/deep-memory");
vi.doMock("@/lib/env", () => ({
env: {
MAX_UPLOAD_MB: 150,
@@ -153,6 +201,7 @@ function mockRuntime(client: ReturnType, ragMock?: Re
RAG_AWAIT_QUERY_LOGS: false,
},
isDemoMode: () => false,
+ isLocalNoAuthMode: () => Boolean(options.localNoAuth),
requireOpenAIEnv: () => undefined,
requireServerEnv: () => undefined,
}));
@@ -168,6 +217,10 @@ function request(path: string, init?: RequestInit) {
return new Request(`http://localhost${path}`, init);
}
+function localPortRequest(port: number, path: string, init?: RequestInit) {
+ return new Request(`http://localhost:${port}${path}`, init);
+}
+
function authenticatedRequest(path: string, init?: RequestInit) {
return request(path, {
...init,
@@ -188,6 +241,59 @@ afterEach(() => {
});
describe("private document API access", () => {
+ it("rejects local no-auth private calls from unmanaged localhost ports before Supabase access", async () => {
+ const client = createSupabaseMock();
+ mockRuntime(client, undefined, { localNoAuth: true });
+ const { GET } = await import("../src/app/api/documents/route");
+
+ const response = await GET(localPortRequest(3000, "/api/documents"));
+
+ expect(response.status).toBe(401);
+ expect(client.auth.getUser).not.toHaveBeenCalled();
+ expect(client.from).not.toHaveBeenCalled();
+ });
+
+ it("rejects managed-port private calls with stale localhost referers before Supabase access", async () => {
+ const client = createSupabaseMock();
+ mockRuntime(client, undefined, { localNoAuth: true });
+ const { GET } = await import("../src/app/api/documents/route");
+
+ const response = await GET(
+ localPortRequest(4298, "/api/documents", {
+ headers: {
+ referer: "http://localhost:3000/",
+ },
+ }),
+ );
+
+ expect(response.status).toBe(401);
+ expect(client.auth.getUser).not.toHaveBeenCalled();
+ expect(client.from).not.toHaveBeenCalled();
+ });
+
+ it("resolves local no-auth owner from documents before listing auth users", async () => {
+ const documents = [{ id: documentId, owner_id: userId, title: "Owned document" }];
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.selected === "owner_id") {
+ return ok({ owner_id: userId });
+ }
+ if (call.table === "documents") {
+ return ok(documents);
+ }
+ return ok([]);
+ });
+ mockRuntime(client, undefined, { localNoAuth: true });
+ const { GET } = await import("../src/app/api/documents/route");
+
+ const response = await GET(localPortRequest(4298, "/api/documents"));
+ const body = await payload(response);
+
+ expect(response.status).toBe(200);
+ expect(body.documents).toEqual(documents.map((document) => ({ ...document, labels: [], summary: null })));
+ expect(client.auth.getUser).not.toHaveBeenCalled();
+ expect(client.calls[0]).toMatchObject({ table: "documents", selected: "owner_id" });
+ });
+
it("rejects unauthenticated document listing", async () => {
const client = createSupabaseMock();
mockRuntime(client);
@@ -211,7 +317,11 @@ describe("private document API access", () => {
expect(response.status).toBe(200);
expect(body.documents).toEqual(documents.map((document) => ({ ...document, labels: [], summary: null })));
+ expect(body.pagination).toMatchObject({ limit: 100, offset: 0, nextOffset: 1, hasMore: false });
expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId });
+ expect(client.calls[0].selected).toContain("id,owner_id,title");
+ expect(client.calls[0].selected).not.toBe("*");
+ expect(client.calls[0].range).toEqual({ from: 0, to: 99 });
});
it("does not return raw internal database errors", async () => {
@@ -348,6 +458,44 @@ describe("private document API access", () => {
expect(client.storageMocks.remove).not.toHaveBeenCalled();
});
+ it("assigns a smart unique title when a different document has the same upload name", async () => {
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.operation === "select" && call.maybeSingle) {
+ return ok(null);
+ }
+ if (call.table === "documents" && call.operation === "select") {
+ return ok([{ id: "existing-doc", title: "Guideline", file_name: "guideline.pdf", content_hash: "other-hash" }]);
+ }
+ if (call.table === "documents" && call.operation === "insert") {
+ const inserted = call.insertPayload as { id: string; title: string; metadata: Record };
+ return ok({ id: inserted.id, title: inserted.title, metadata: inserted.metadata });
+ }
+ if (call.table === "ingestion_jobs" && call.operation === "insert") {
+ return ok({ id: "job-1", document_id: documentId });
+ }
+ return ok([]);
+ });
+ mockRuntime(client);
+ const { POST } = await import("../src/app/api/upload/route");
+ const formData = new FormData();
+ formData.set("file", new File(["%PDF-1.7 revised"], "guideline.pdf", { type: "application/pdf" }));
+
+ const response = await POST(
+ authenticatedRequest("/api/upload", {
+ method: "POST",
+ body: formData,
+ }),
+ );
+ const documentInsert = client.calls.find((call) => call.table === "documents" && call.operation === "insert");
+ const inserted = documentInsert?.insertPayload as { title: string; metadata: Record };
+
+ expect(response.status).toBe(201);
+ expect(inserted.title).toBe("Guideline (Copy 2)");
+ expect(inserted.metadata.smart_title_base).toBe("Guideline");
+ expect(inserted.metadata.smart_title_duplicate_reason).toBe("same_title_or_filename");
+ expect(inserted.metadata.smart_title_duplicate_index).toBe(2);
+ });
+
it("alerts on exact-copy uploads without storing or queueing a duplicate", async () => {
const duplicate = {
id: documentId,
@@ -385,6 +533,214 @@ describe("private document API access", () => {
expect(client.calls.some((call) => call.table === "ingestion_jobs" && call.operation === "insert")).toBe(false);
});
+ it("returns first-class document matches for document-focused search", async () => {
+ const client = createSupabaseMock((call) => {
+ if (call.table === "document_labels") {
+ return ok([
+ {
+ id: "label-1",
+ document_id: documentId,
+ label: "agitation",
+ label_type: "topic",
+ source: "generated",
+ confidence: 0.9,
+ },
+ ]);
+ }
+ if (call.table === "document_summaries") {
+ return ok([{ document_id: documentId, summary: "High-yield agitation management guidance." }]);
+ }
+ if (call.table === "document_images") {
+ return ok([
+ {
+ document_id: documentId,
+ source_kind: "table_crop",
+ searchable: true,
+ image_type: "clinical_table",
+ clinical_relevance_score: 0.9,
+ metadata: { clinical_use_class: "clinical_evidence" },
+ },
+ {
+ document_id: documentId,
+ source_kind: "embedded",
+ searchable: true,
+ image_type: "graph",
+ clinical_relevance_score: 0.82,
+ metadata: { clinical_use_class: "clinical_evidence" },
+ },
+ ]);
+ }
+ return ok([]);
+ });
+ client.rpc.mockResolvedValue({
+ data: [
+ {
+ document_id: documentId,
+ labels: [
+ {
+ id: "label-1",
+ document_id: documentId,
+ label: "agitation",
+ label_type: "topic",
+ source: "generated",
+ confidence: 0.9,
+ },
+ ],
+ summary: "High-yield agitation management guidance.",
+ },
+ ],
+ error: null,
+ });
+ mockRuntime(client, {
+ searchChunksWithTelemetry: vi.fn(async () => ({
+ results: [
+ {
+ id: "chunk-1",
+ document_id: documentId,
+ title: "Agitation and arousal",
+ file_name: "MHSP.AgitationArousalPharmaMgt.pdf",
+ page_number: 3,
+ chunk_index: 0,
+ section_heading: "Management",
+ content: "Agitation management table text.",
+ image_ids: [],
+ similarity: 0.9,
+ hybrid_score: 0.92,
+ images: [],
+ },
+ ],
+ telemetry: {
+ retrieval_strategy: "text",
+ search_cache_hit: false,
+ embedding_skipped: true,
+ embedding_cache_hit: false,
+ text_fast_path_latency_ms: 10,
+ embedding_latency_ms: 0,
+ supabase_rpc_latency_ms: 12,
+ rerank_latency_ms: 1,
+ },
+ })),
+ });
+ const { POST } = await import("../src/app/api/search/route");
+
+ const response = await POST(
+ request("/api/search", {
+ method: "POST",
+ body: JSON.stringify({ query: "agitation management tables", mode: "documents", documentLimit: 10 }),
+ }),
+ );
+ const body = await payload(response);
+
+ expect(response.status).toBe(200);
+ expect(body.documentMatches).toEqual([
+ expect.objectContaining({
+ document_id: documentId,
+ title: "Agitation and arousal",
+ bestPages: [3],
+ tableCount: 1,
+ imageCount: 2,
+ summarySnippet: "High-yield agitation management guidance.",
+ }),
+ ]);
+ expect(body.relatedDocuments).toHaveLength(1);
+ });
+
+ it("rejects unauthenticated reindex requests", async () => {
+ const client = createSupabaseMock();
+ mockRuntime(client);
+ const { POST } = await import("../src/app/api/documents/[id]/reindex/route");
+
+ const response = await POST(
+ request(`/api/documents/${documentId}/reindex`, {
+ method: "POST",
+ body: JSON.stringify({ mode: "enrichment" }),
+ }),
+ { params: Promise.resolve({ id: documentId }) },
+ );
+
+ expect(response.status).toBe(401);
+ expect(await payload(response)).toEqual({ error: "Authentication required." });
+ expect(client.from).not.toHaveBeenCalled();
+ });
+
+ it("runs enrichment-only reindex for owned indexed documents using generic metadata", async () => {
+ const document = {
+ id: documentId,
+ owner_id: userId,
+ title: "Future Uploaded Protocol",
+ file_name: "future-upload.pdf",
+ source_path: null,
+ import_batch_id: null,
+ metadata: { existing: true },
+ };
+ const chunks = [
+ {
+ id: "chunk-1",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: "Workflow",
+ content: "Future uploaded protocol workflow content.",
+ },
+ ];
+ const images = [
+ {
+ id: imageId,
+ page_number: 1,
+ caption: "Clinical workflow table.",
+ image_type: "clinical_table",
+ labels: ["workflow"],
+ },
+ ];
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.operation === "select") return ok(document);
+ if (call.table === "document_chunks") return ok(chunks);
+ if (call.table === "document_images") return ok(images);
+ return ok([]);
+ });
+ const upsertDocumentEnrichment = vi.fn(async () => ({
+ summary: { id: "summary-1", document_id: documentId, summary: "Source-backed summary." },
+ labels: [],
+ }));
+ const upsertDocumentDeepMemory = vi.fn(async () => ({
+ sections: [],
+ memoryCards: [],
+ }));
+ mockRuntime(client);
+ vi.doMock("@/lib/document-enrichment", () => ({ upsertDocumentEnrichment }));
+ vi.doMock("@/lib/deep-memory", () => ({ upsertDocumentDeepMemory }));
+ const { POST } = await import("../src/app/api/documents/[id]/reindex/route");
+
+ const response = await POST(
+ authenticatedRequest(`/api/documents/${documentId}/reindex`, {
+ method: "POST",
+ body: JSON.stringify({ mode: "enrichment" }),
+ }),
+ { params: Promise.resolve({ id: documentId }) },
+ );
+
+ expect(response.status).toBe(200);
+ expect(upsertDocumentEnrichment).toHaveBeenCalledWith(
+ expect.objectContaining({
+ document: expect.objectContaining({
+ id: documentId,
+ title: "Future Uploaded Protocol",
+ metadata: { existing: true },
+ }),
+ chunks,
+ images,
+ }),
+ );
+ expect(client.calls[0].selected).toContain("metadata");
+ expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId });
+ expect(upsertDocumentDeepMemory).toHaveBeenCalledWith(
+ expect.objectContaining({
+ document: expect.objectContaining({ id: documentId }),
+ chunks,
+ images,
+ }),
+ );
+ });
+
it("cleans up uploaded storage when document insert fails", async () => {
const client = createSupabaseMock((call) =>
call.table === "documents" && call.operation === "insert" ? fail("document insert failed") : ok([]),
@@ -447,8 +803,165 @@ describe("private document API access", () => {
expect(client.calls).toHaveLength(1);
});
- it("passes owner scope through search and answer routes", async () => {
- const searchChunksWithTelemetry = vi.fn(async () => ({
+ it("allows owners to rename the document display title without changing file provenance", async () => {
+ const original = {
+ id: documentId,
+ owner_id: userId,
+ title: "Original title",
+ file_name: "source.pdf",
+ storage_path: `${userId}/documents/${documentId}/source.pdf`,
+ content_hash: "hash-1",
+ metadata: { source_path: "C:\\Guidelines\\source.pdf" },
+ };
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.operation === "select") return ok(original);
+ if (call.table === "documents" && call.operation === "update") {
+ return ok({
+ ...original,
+ ...(call.updatePayload as Record),
+ });
+ }
+ return ok([]);
+ });
+ mockRuntime(client);
+ const { PATCH } = await import("../src/app/api/documents/[id]/route");
+
+ const response = await PATCH(
+ authenticatedRequest(`/api/documents/${documentId}`, {
+ method: "PATCH",
+ body: JSON.stringify({ title: "Better clinical title" }),
+ }),
+ { params: Promise.resolve({ id: documentId }) },
+ );
+ const body = await payload(response);
+ const updateCall = client.calls.find((call) => call.table === "documents" && call.operation === "update");
+ const update = updateCall?.updatePayload as Record;
+
+ expect(response.status).toBe(200);
+ expect((body.document as Record).title).toBe("Better clinical title");
+ expect(update.title).toBe("Better clinical title");
+ expect(update).not.toHaveProperty("file_name");
+ expect(update).not.toHaveProperty("storage_path");
+ expect(update).not.toHaveProperty("content_hash");
+ expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId });
+ });
+
+ it("rejects invalid document rename titles", async () => {
+ const client = createSupabaseMock();
+ mockRuntime(client);
+ const { PATCH } = await import("../src/app/api/documents/[id]/route");
+
+ const response = await PATCH(
+ authenticatedRequest(`/api/documents/${documentId}`, {
+ method: "PATCH",
+ body: JSON.stringify({ title: " " }),
+ }),
+ { params: Promise.resolve({ id: documentId }) },
+ );
+
+ expect(response.status).toBe(400);
+ expect(await payload(response)).toEqual({ error: "Enter a document title between 1 and 180 characters." });
+ expect(client.from).not.toHaveBeenCalled();
+ });
+
+ it("does not rename another user's document", async () => {
+ const client = createSupabaseMock(() => ok(null));
+ mockRuntime(client);
+ const { PATCH } = await import("../src/app/api/documents/[id]/route");
+
+ const response = await PATCH(
+ authenticatedRequest(`/api/documents/${otherDocumentId}`, {
+ method: "PATCH",
+ body: JSON.stringify({ title: "Not mine" }),
+ }),
+ { params: Promise.resolve({ id: otherDocumentId }) },
+ );
+
+ expect(response.status).toBe(404);
+ expect(await payload(response)).toEqual({ error: "Document not found." });
+ expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId });
+ });
+
+ it("permanently deletes an owned document, query logs, and storage objects", async () => {
+ const sourcePath = `${userId}/documents/${documentId}/source.pdf`;
+ const imagePath = `${userId}/images/${imageId}.png`;
+ const chunkId = "44444444-4444-4444-8444-444444444444";
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.operation === "select") {
+ return ok({ id: documentId, owner_id: userId, title: "Owned", storage_path: sourcePath });
+ }
+ if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]);
+ if (call.table === "document_images" && call.operation === "select") return ok([{ storage_path: imagePath }]);
+ if (call.table === "document_chunks" && call.operation === "select") return ok([{ id: chunkId }]);
+ if (call.table === "storage_cleanup_jobs" && call.operation === "insert") return ok({ id: "cleanup-1" });
+ if (call.table === "storage_cleanup_jobs" && call.operation === "update") return ok([]);
+ if (call.table === "rag_queries" && call.operation === "delete") return ok([]);
+ if (call.table === "documents" && call.operation === "delete") return ok([]);
+ return ok([]);
+ });
+ mockRuntime(client);
+ const { DELETE } = await import("../src/app/api/documents/[id]/route");
+
+ const response = await DELETE(authenticatedRequest(`/api/documents/${documentId}`, { method: "DELETE" }), {
+ params: Promise.resolve({ id: documentId }),
+ });
+ const body = await payload(response);
+ const ragDelete = client.calls.find((call) => call.table === "rag_queries" && call.operation === "delete");
+ const documentDelete = client.calls.find((call) => call.table === "documents" && call.operation === "delete");
+ const cleanupInsert = client.calls.find(
+ (call) => call.table === "storage_cleanup_jobs" && call.operation === "insert",
+ );
+ const cleanupUpdate = client.calls.find(
+ (call) => call.table === "storage_cleanup_jobs" && call.operation === "update",
+ );
+
+ expect(response.status).toBe(200);
+ expect(body).toMatchObject({ deleted: true, documentId, storageWarnings: [] });
+ expect(cleanupInsert?.insertPayload).toMatchObject({
+ owner_id: userId,
+ document_id: documentId,
+ document_paths: [sourcePath],
+ image_paths: [imagePath],
+ status: "pending",
+ });
+ expect(cleanupUpdate?.updatePayload).toMatchObject({ status: "completed", storage_removed: 0 });
+ expect(ragDelete?.filters).toContainEqual({ column: "owner_id", value: userId });
+ expect(ragDelete?.overlapsFilters).toContainEqual({ column: "source_chunk_ids", values: [chunkId] });
+ expect(documentDelete?.filters).toContainEqual({ column: "id", value: documentId });
+ expect(documentDelete?.filters).toContainEqual({ column: "owner_id", value: userId });
+ expect(client.storageMocks.storageFrom).toHaveBeenCalledWith("clinical-documents");
+ expect(client.storageMocks.storageFrom).toHaveBeenCalledWith("clinical-images");
+ expect(client.storageMocks.remove).toHaveBeenCalledWith([sourcePath]);
+ expect(client.storageMocks.remove).toHaveBeenCalledWith([imagePath]);
+ });
+
+ it("blocks permanent delete while a document is actively indexing", async () => {
+ const client = createSupabaseMock((call) => {
+ if (call.table === "documents" && call.operation === "select") {
+ return ok({ id: documentId, owner_id: userId, title: "Owned", storage_path: "source.pdf" });
+ }
+ if (call.table === "ingestion_jobs" && call.operation === "select") {
+ return ok([{ id: "job-1", status: "processing" }]);
+ }
+ return ok([]);
+ });
+ mockRuntime(client);
+ const { DELETE } = await import("../src/app/api/documents/[id]/route");
+
+ const response = await DELETE(authenticatedRequest(`/api/documents/${documentId}`, { method: "DELETE" }), {
+ params: Promise.resolve({ id: documentId }),
+ });
+
+ expect(response.status).toBe(409);
+ expect(await payload(response)).toEqual({
+ error: "Document is currently indexing. Stop or wait for the worker before deleting.",
+ });
+ expect(client.calls.some((call) => call.table === "documents" && call.operation === "delete")).toBe(false);
+ expect(client.storageMocks.remove).not.toHaveBeenCalled();
+ });
+
+ it("allows unauthenticated search and answer without owner scope", async () => {
+ const searchChunksWithTelemetry = vi.fn(async (_args: unknown) => ({
results: [],
telemetry: {
search_cache_hit: false,
@@ -461,7 +974,7 @@ describe("private document API access", () => {
retrieval_strategy: "text_fast_path",
},
}));
- const answerQuestionWithScope = vi.fn(async () => ({
+ const answerQuestionWithScope = vi.fn(async (_args: unknown) => ({
answer: "No owned evidence.",
grounded: false,
confidence: "unsupported",
@@ -475,27 +988,184 @@ describe("private document API access", () => {
const answerRoute = await import("../src/app/api/answer/route");
await searchRoute.POST(
- authenticatedRequest("/api/search", {
+ request("/api/search", {
method: "POST",
body: JSON.stringify({ query: "monitoring", documentIds: [otherDocumentId] }),
}),
);
await answerRoute.POST(
- authenticatedRequest("/api/answer", {
+ request("/api/answer", {
method: "POST",
body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }),
}),
);
expect(searchChunksWithTelemetry).toHaveBeenCalledWith(
- expect.objectContaining({ ownerId: userId, documentIds: [otherDocumentId] }),
+ expect.objectContaining({ ownerId: undefined, documentIds: [otherDocumentId] }),
);
expect(answerQuestionWithScope).toHaveBeenCalledWith(
- expect.objectContaining({ ownerId: userId, documentId: otherDocumentId }),
+ expect.objectContaining({ ownerId: undefined, documentId: otherDocumentId }),
+ );
+ expect(client.auth.getUser).not.toHaveBeenCalled();
+ });
+
+ it("rate limits public answer generation without limiting public search", async () => {
+ const searchChunksWithTelemetry = vi.fn(async (_args: unknown) => ({
+ results: [],
+ telemetry: {
+ search_cache_hit: false,
+ text_fast_path_latency_ms: 0,
+ embedding_skipped: true,
+ embedding_latency_ms: 0,
+ embedding_cache_hit: false,
+ supabase_rpc_latency_ms: 0,
+ rerank_latency_ms: 0,
+ retrieval_strategy: "text_fast_path",
+ },
+ }));
+ const answerQuestionWithScope = vi.fn(async (_args: unknown) => ({
+ answer: "Source-backed answer.",
+ grounded: true,
+ confidence: "medium",
+ citations: [],
+ sources: [],
+ }));
+ const client = createSupabaseMock();
+ mockRuntime(client, { searchChunksWithTelemetry, answerQuestionWithScope });
+
+ const answerRoute = await import("../src/app/api/answer/route");
+ const searchRoute = await import("../src/app/api/search/route");
+ const answerRequest = () =>
+ request("/api/answer", {
+ method: "POST",
+ headers: { "x-forwarded-for": "203.0.113.10" },
+ body: JSON.stringify({ query: "monitoring" }),
+ });
+
+ for (let index = 0; index < 30; index += 1) {
+ const response = await answerRoute.POST(answerRequest());
+ expect(response.status).toBe(200);
+ }
+
+ const limited = await answerRoute.POST(answerRequest());
+ expect(limited.status).toBe(429);
+ expect(limited.headers.get("Retry-After")).toBe("60");
+ expect(await payload(limited)).toEqual({ error: "Too many public answer requests. Retry shortly." });
+ expect(answerQuestionWithScope).toHaveBeenCalledTimes(30);
+
+ const searchResponse = await searchRoute.POST(
+ request("/api/search", {
+ method: "POST",
+ headers: { "x-forwarded-for": "203.0.113.10" },
+ body: JSON.stringify({ query: "monitoring" }),
+ }),
);
+
+ expect(searchResponse.status).toBe(200);
+ expect(searchChunksWithTelemetry).toHaveBeenCalledWith(expect.objectContaining({ ownerId: undefined }));
});
- it("streams answer progress before the final answer with owner scope", async () => {
+ it("rate limits abnormal public search bursts with retry metadata", async () => {
+ const searchChunksWithTelemetry = vi.fn(async (_args: unknown) => ({
+ results: [],
+ telemetry: {
+ search_cache_hit: false,
+ text_fast_path_latency_ms: 0,
+ embedding_skipped: true,
+ embedding_latency_ms: 0,
+ embedding_cache_hit: false,
+ supabase_rpc_latency_ms: 0,
+ rerank_latency_ms: 0,
+ retrieval_strategy: "text_fast_path",
+ weighted_top_score: 0,
+ rrf_top_score: 0,
+ },
+ }));
+ const client = createSupabaseMock();
+ mockRuntime(client, { searchChunksWithTelemetry });
+ vi.doMock("@/lib/public-rate-limit", async () => {
+ const actual = await vi.importActual(
+ "@/lib/public-rate-limit",
+ );
+ return {
+ ...actual,
+ consumePublicSearchRateLimit: (headers: Headers) =>
+ actual.consumePublicSearchRateLimit(headers, Date.now(), { limit: 2, windowMs: 60_000 }),
+ };
+ });
+ const searchRoute = await import("../src/app/api/search/route");
+ const searchRequest = () =>
+ request("/api/search", {
+ method: "POST",
+ headers: { "x-forwarded-for": "203.0.113.20" },
+ body: JSON.stringify({ query: "monitoring", includeRelatedDocuments: false }),
+ });
+
+ expect((await searchRoute.POST(searchRequest())).status).toBe(200);
+ expect((await searchRoute.POST(searchRequest())).status).toBe(200);
+
+ const limited = await searchRoute.POST(searchRequest());
+ expect(limited.status).toBe(429);
+ expect(limited.headers.get("Retry-After")).toBe("60");
+ expect(await payload(limited)).toEqual({
+ error: "Search is temporarily rate limited because too many requests were received. Retry shortly.",
+ retryAfterSeconds: 60,
+ });
+ expect(searchChunksWithTelemetry).toHaveBeenCalledTimes(2);
+ });
+
+ it("coalesces identical in-flight public search requests", async () => {
+ let releaseSearch!: () => void;
+ const searchGate = new Promise((resolve) => {
+ releaseSearch = resolve;
+ });
+ const searchChunksWithTelemetry = vi.fn(async (_args: unknown) => {
+ await searchGate;
+ return {
+ results: [],
+ telemetry: {
+ search_cache_hit: false,
+ text_fast_path_latency_ms: 0,
+ embedding_skipped: true,
+ embedding_latency_ms: 0,
+ embedding_cache_hit: false,
+ supabase_rpc_latency_ms: 0,
+ rerank_latency_ms: 0,
+ retrieval_strategy: "text_fast_path",
+ weighted_top_score: 0,
+ rrf_top_score: 0,
+ },
+ };
+ });
+ const client = createSupabaseMock();
+ mockRuntime(client, { searchChunksWithTelemetry });
+ const searchRoute = await import("../src/app/api/search/route");
+ const searchRequest = () =>
+ request("/api/search", {
+ method: "POST",
+ headers: { "x-real-ip": "203.0.113.21" },
+ body: JSON.stringify({ query: "monitoring", includeRelatedDocuments: false }),
+ });
+
+ const first = searchRoute.POST(searchRequest());
+ const second = searchRoute.POST(searchRequest());
+ for (let index = 0; index < 10 && searchChunksWithTelemetry.mock.calls.length === 0; index += 1) {
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ }
+ expect(searchChunksWithTelemetry).toHaveBeenCalledTimes(1);
+
+ releaseSearch();
+ const [firstResponse, secondResponse] = await Promise.all([first, second]);
+ const firstPayload = await payload(firstResponse);
+ const secondPayload = await payload(secondResponse);
+
+ expect(firstResponse.status).toBe(200);
+ expect(secondResponse.status).toBe(200);
+ expect(firstPayload.telemetry).toMatchObject({ coalesced: false });
+ expect(secondPayload.telemetry).toMatchObject({ coalesced: true });
+ });
+
+ it("streams answer progress before the final answer without auth", async () => {
const answerQuestionWithScope = vi.fn(async (args: { onProgress?: (event: unknown) => void | Promise }) => {
await args.onProgress?.({ stage: "retrieved", message: "Retrieved 2 candidate sources." });
return {
@@ -511,7 +1181,7 @@ describe("private document API access", () => {
const { POST } = await import("../src/app/api/answer/stream/route");
const response = await POST(
- authenticatedRequest("/api/answer/stream", {
+ request("/api/answer/stream", {
method: "POST",
body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }),
}),
@@ -523,8 +1193,51 @@ describe("private document API access", () => {
expect(body.indexOf("event: final")).toBeGreaterThan(body.indexOf("event: progress"));
expect(body).toContain("Retrieved 2 candidate sources.");
expect(answerQuestionWithScope).toHaveBeenCalledWith(
- expect.objectContaining({ ownerId: userId, documentId: otherDocumentId, onProgress: expect.any(Function) }),
+ expect.objectContaining({ ownerId: undefined, documentId: otherDocumentId, onProgress: expect.any(Function) }),
+ );
+ expect(client.auth.getUser).not.toHaveBeenCalled();
+ });
+
+ it("emits a structured SSE error when public streaming answers are rate limited", async () => {
+ const answerQuestionWithScope = vi.fn(async () => ({
+ answer: "Owned evidence.",
+ grounded: true,
+ confidence: "medium",
+ citations: [],
+ sources: [],
+ }));
+ const client = createSupabaseMock();
+ mockRuntime(client, { answerQuestionWithScope });
+
+ const answerRoute = await import("../src/app/api/answer/route");
+ const streamRoute = await import("../src/app/api/answer/stream/route");
+ const answerRequest = () =>
+ request("/api/answer", {
+ method: "POST",
+ headers: { "x-real-ip": "203.0.113.11" },
+ body: JSON.stringify({ query: "monitoring" }),
+ });
+
+ for (let index = 0; index < 30; index += 1) {
+ const response = await answerRoute.POST(answerRequest());
+ expect(response.status).toBe(200);
+ }
+
+ const response = await streamRoute.POST(
+ request("/api/answer/stream", {
+ method: "POST",
+ headers: { "x-real-ip": "203.0.113.11" },
+ body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }),
+ }),
);
+ const body = await response.text();
+
+ expect(response.status).toBe(429);
+ expect(response.headers.get("Retry-After")).toBe("60");
+ expect(body).toContain("event: error");
+ expect(body).toContain('"status":429');
+ expect(body).toContain("Too many public answer requests. Retry shortly.");
+ expect(answerQuestionWithScope).toHaveBeenCalledTimes(30);
});
it("returns a generic not found response when summarizing an unowned document", async () => {
@@ -575,4 +1288,29 @@ describe("private document API access", () => {
}),
);
});
+
+ it("uses the DB-backed document lookup RPC with owner scope", async () => {
+ const client = createSupabaseMock();
+ mockRuntime(client);
+ const embedTextWithTelemetry = vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false }));
+ vi.doMock("@/lib/openai", () => ({
+ embedTextWithTelemetry,
+ generateTextResponse: vi.fn(),
+ generateStructuredTextResponse: vi.fn(),
+ generateStructuredTextResult: vi.fn(),
+ }));
+ const { searchChunks } = await import("../src/lib/rag");
+
+ await searchChunks({
+ query: "Find the NOCC document",
+ ownerId: userId,
+ });
+
+ expect(client.rpc).toHaveBeenCalledWith(
+ "match_documents_for_query",
+ expect.objectContaining({
+ owner_filter: userId,
+ }),
+ );
+ });
});
diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts
new file mode 100644
index 000000000..670139f4d
--- /dev/null
+++ b/tests/rag-answer-fallback.test.ts
@@ -0,0 +1,266 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { SearchResult } from "../src/lib/types";
+
+function source(overrides: Partial = {}): SearchResult {
+ return {
+ id: "agitation-chunk-1",
+ document_id: "agitation-doc",
+ title: "Agitation and Arousal Pharmacological Management",
+ file_name: "MHSP.AgitationArousalPharmaMgt.pdf",
+ page_number: 8,
+ chunk_index: 0,
+ section_heading: "Appendix 1",
+ content:
+ "Agitation and arousal pharmacological management for adult mental health inpatients. Step 1 uses oral medication when the patient is willing. Step 2 considers increased agitation and arousal ratings with oral benzodiazepines or antipsychotics. Step 3 uses intramuscular medication when oral medication is refused.",
+ image_ids: [],
+ similarity: 0.97,
+ hybrid_score: 0.97,
+ text_rank: 1.1,
+ source_metadata: {
+ source_title: "Agitation source",
+ publisher: "Local service",
+ jurisdiction: "Australia/WA",
+ version: "1",
+ publication_date: null,
+ review_date: null,
+ uploaded_at: null,
+ indexed_at: null,
+ uploaded_by: null,
+ document_status: "current",
+ clinical_validation_status: "approved",
+ extraction_quality: "good",
+ },
+ images: [],
+ ...overrides,
+ };
+}
+
+class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> {
+ select() {
+ return this;
+ }
+
+ in() {
+ return this;
+ }
+
+ eq() {
+ return this;
+ }
+
+ neq() {
+ return this;
+ }
+
+ order() {
+ return this;
+ }
+
+ limit() {
+ return Promise.resolve({ data: [], error: null });
+ }
+
+ then(
+ onfulfilled?: ((value: { data: unknown[]; error: null }) => TResult1 | PromiseLike) | null,
+ onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null,
+ ): PromiseLike {
+ return Promise.resolve({ data: [], error: null }).then(onfulfilled, onrejected);
+ }
+}
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.resetModules();
+ vi.unstubAllEnvs();
+});
+
+describe("RAG structured-output fallback", () => {
+ it("returns natural extractive answers instead of packed source-card labels", async () => {
+ vi.stubEnv("OPENAI_API_KEY", "test-key");
+ vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0");
+ vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0");
+
+ const clozapineSource = source({
+ id: "clozapine-monitoring-1",
+ document_id: "clozapine-doc",
+ title: "Clozapine Monitoring",
+ file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf",
+ page_number: 11,
+ section_heading: "Monitoring",
+ content:
+ "Medication point: • Copy of the Consent to Clozapine Treatment Form EMR0270. Medication point: • Prescribe initiation of Clozapine on the WA Adult Clozapine Initiation and Titration form. Medication point: • Ensure consumers complete the Clozapine Monitoring Form on initiation.",
+ similarity: 0.94,
+ hybrid_score: 0.94,
+ text_rank: 1.2,
+ memory_cards: [
+ {
+ id: "memory-1",
+ document_id: "clozapine-doc",
+ owner_id: null,
+ card_type: "medication",
+ title: "Clozapine monitoring",
+ content:
+ "Medication point: • Copy of the Consent to Clozapine Treatment Form EMR0270 - Medication point: • Prescribe Initiation of Clozapine on the WA Adult Clozapine Initiation and Titration form - Medication point: • Ensure all consumers complete the Clozapine Monitoring Form on initiation.",
+ normalized_terms: ["clozapine", "monitoring"],
+ page_number: 11,
+ source_chunk_ids: ["clozapine-monitoring-1"],
+ source_image_ids: [],
+ confidence: 0.92,
+ },
+ ],
+ });
+ const rpc = vi.fn(async (name: string) => {
+ if (name === "match_document_chunks_text") return { data: [clozapineSource], error: null };
+ if (name === "get_related_document_metadata") return { data: [], error: null };
+ return { data: [], error: null };
+ });
+
+ vi.doMock("@/lib/supabase/admin", () => ({
+ createAdminClient: () => ({
+ rpc,
+ from: vi.fn(() => new EmptyQuery()),
+ }),
+ }));
+ vi.doMock("@/lib/openai", () => ({
+ embedTextWithTelemetry: vi.fn(),
+ generateStructuredTextResult: vi.fn(),
+ }));
+
+ const { answerQuestionWithScope } = await import("../src/lib/rag");
+
+ const answer = await answerQuestionWithScope({
+ query: "what monitoring is required for clozapine",
+ ownerId: undefined,
+ logQuery: false,
+ skipCache: true,
+ });
+
+ expect(answer.routingMode).toBe("extractive");
+ expect(answer.answer.replace(/\*\*/g, "")).toContain("retrieved clozapine sources");
+ expect(answer.answer.replace(/\*\*/g, "")).toContain("Clozapine Monitoring Form");
+ expect(answer.answer).not.toContain("- Medication point");
+ expect(answer.answer).not.toMatch(/Medication point:.*Medication point:/);
+ expect(answer.answerSections?.[0]?.heading).toBe("Direct source-backed answer");
+ expect(answer.answerSections?.[0]?.body).not.toContain("- Medication point");
+ });
+
+ it("returns an extractive source-backed answer when structured model output is truncated", async () => {
+ vi.stubEnv("OPENAI_API_KEY", "test-key");
+ vi.stubEnv("OPENAI_MAX_OUTPUT_TOKENS", "650");
+ vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0");
+ vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0");
+
+ const sources = [source(), source({ id: "agitation-chunk-2", page_number: 10, chunk_index: 1 })];
+ const rpc = vi.fn(async (name: string) => {
+ if (name === "match_document_chunks_text") return { data: sources, error: null };
+ if (name === "get_related_document_metadata") return { data: [], error: null };
+ return { data: [], error: null };
+ });
+ const generateStructuredTextResult = vi.fn(async () => ({
+ text: '{"answer":"Agitation and arousal management starts',
+ model: "gpt-4.1-mini",
+ operation: "answer",
+ latencyMs: 12,
+ requestId: "req_truncated",
+ usage: { input_tokens: 100, output_tokens: 650, total_tokens: 750 },
+ }));
+
+ vi.doMock("@/lib/supabase/admin", () => ({
+ createAdminClient: () => ({
+ rpc,
+ from: vi.fn(() => new EmptyQuery()),
+ }),
+ }));
+ vi.doMock("@/lib/openai", () => ({
+ embedTextWithTelemetry: vi.fn(),
+ generateStructuredTextResult,
+ }));
+
+ const { answerQuestionWithScope, machineReadableFallbackAnswer } = await import("../src/lib/rag");
+
+ const answer = await answerQuestionWithScope({
+ query: "Summarize inpatient approach",
+ ownerId: undefined,
+ logQuery: false,
+ skipCache: true,
+ });
+
+ expect(answer.answer).not.toBe(machineReadableFallbackAnswer);
+ expect(answer.answer.toLowerCase()).toContain("agitation and arousal");
+ expect(answer.grounded).toBe(true);
+ expect(answer.citations.length).toBeGreaterThan(0);
+ expect(answer.quoteCards?.length).toBeGreaterThan(0);
+ expect(answer.routingMode).toBe("extractive");
+ expect(answer.routingReason).toContain("structured_output_fallback");
+ expect(answer.openAIRequestIds).toEqual(["req_truncated"]);
+ expect(answer.openAIUsage).toMatchObject({ output_tokens: 650 });
+ expect(answer.citations[0]?.source_metadata?.document_status).toBe("current");
+ });
+
+ it("keeps valid structured model answers on the generated-answer path", async () => {
+ vi.stubEnv("OPENAI_API_KEY", "test-key");
+ vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0");
+ vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0");
+
+ const sources = [source()];
+ const rpc = vi.fn(async (name: string) => {
+ if (name === "match_document_chunks_text") return { data: sources, error: null };
+ if (name === "get_related_document_metadata") return { data: [], error: null };
+ return { data: [], error: null };
+ });
+ const generateStructuredTextResult = vi.fn(async () => ({
+ text: JSON.stringify({
+ answer: "Use a stepwise agitation and arousal approach based on rating and route.",
+ grounded: true,
+ confidence: "high",
+ answerSections: [
+ {
+ heading: "Bottom line",
+ body: "Use a stepwise agitation and arousal approach based on rating and route.",
+ citation_chunk_ids: ["agitation-chunk-1"],
+ },
+ ],
+ citations: [{ chunk_id: "agitation-chunk-1" }],
+ quoteCards: [
+ {
+ chunk_id: "agitation-chunk-1",
+ quote: "Agitation and arousal pharmacological management for adult mental health inpatients.",
+ section_heading: "Appendix 1",
+ },
+ ],
+ conflictsOrGaps: [],
+ }),
+ model: "gpt-4.1-mini",
+ operation: "answer",
+ latencyMs: 12,
+ requestId: "req_valid",
+ usage: { input_tokens: 100, output_tokens: 120, total_tokens: 220 },
+ }));
+
+ vi.doMock("@/lib/supabase/admin", () => ({
+ createAdminClient: () => ({
+ rpc,
+ from: vi.fn(() => new EmptyQuery()),
+ }),
+ }));
+ vi.doMock("@/lib/openai", () => ({
+ embedTextWithTelemetry: vi.fn(),
+ generateStructuredTextResult,
+ }));
+
+ const { answerQuestionWithScope } = await import("../src/lib/rag");
+
+ const answer = await answerQuestionWithScope({
+ query: "Summarize inpatient approach",
+ ownerId: undefined,
+ logQuery: false,
+ skipCache: true,
+ });
+
+ expect(answer.answer).toBe("Use a stepwise agitation and arousal approach based on rating and route.");
+ expect(answer.routingMode).toBe("fast");
+ expect(answer.routingReason).not.toContain("structured_output_fallback");
+ expect(answer.openAIRequestIds).toEqual(["req_valid"]);
+ expect(answer.quoteCards?.length).toBe(1);
+ });
+});
diff --git a/tests/rag-routing.test.ts b/tests/rag-routing.test.ts
index 17293ce74..b4659322d 100644
--- a/tests/rag-routing.test.ts
+++ b/tests/rag-routing.test.ts
@@ -30,12 +30,12 @@ function route(query: string, results: SearchResult[]) {
}
describe("RAG answer routing", () => {
- it("uses the extractive route for direct routine questions with strong retrieval", () => {
+ it("uses extractive answers for direct routine document questions with strong retrieval", () => {
const selected = route("What does the admission information document include?", [source()]);
expect(selected.mode).toBe("extractive");
expect(selected.model).toBeNull();
- expect(selected.reason).toBe("strong_source_match_extract");
+ expect(selected.reason).toBe("document_lookup_source_extractive");
});
it("uses the fast model for broader routine questions with strong retrieval", () => {
@@ -46,12 +46,12 @@ describe("RAG answer routing", () => {
expect(selected.reason).toBe("strong_routine_retrieval");
});
- it("uses the extractive route for routine medication questions with strong source support", () => {
+ it("uses extractive answers for routine medication questions with strong source support", () => {
const selected = route("What clozapine monitoring is required?", [source()]);
expect(selected.mode).toBe("extractive");
expect(selected.model).toBeNull();
- expect(selected.reason).toBe("strong_source_match_extract");
+ expect(selected.reason).toBe("high_confidence_source_extractive");
});
it("uses the strong model for medication or risk-heavy decision questions", () => {
@@ -69,7 +69,7 @@ describe("RAG answer routing", () => {
expect(selected.reason).toBe("limited_retrieval_strength");
});
- it("keeps direct title matches on the extractive path when the question is routine", () => {
+ it("keeps direct title matches on the fast synthesis path when the question is routine", () => {
const selected = chooseAnswerRoute({
query: "What are NOCC requirements?",
results: [
@@ -86,7 +86,44 @@ describe("RAG answer routing", () => {
});
expect(selected.mode).toBe("extractive");
- expect(selected.reason).toBe("strong_source_match_extract");
+ expect(selected.reason).toBe("high_confidence_source_extractive");
+ });
+
+ it("skips generation for document lookups without direct title support", () => {
+ const selected = route("Find the newly uploaded Future Synthetic Ketamine Sedation Protocol.", [
+ source({
+ title: "Agitation and Arousal Pharmacological Management",
+ file_name: "MHSP.AgitationArousalPharmaMgt.pdf",
+ content: "Ketamine sedation may be discussed in a table row.",
+ similarity: 0.5,
+ hybrid_score: 0.42,
+ text_rank: 0.01,
+ }),
+ ]);
+
+ expect(selected.mode).toBe("unsupported");
+ expect(selected.reason).toBe("document_lookup_without_title_support");
+ expect(selected.model).toBeNull();
+ });
+
+ it("uses extractive answers for direct table or threshold lookups with strong source support", () => {
+ const selected = route("What ANC threshold should stop clozapine?", [
+ source({
+ title: "Clozapine Prescribing and Monitoring",
+ file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf",
+ text_rank: 0.08,
+ }),
+ ]);
+
+ expect(selected.mode).toBe("extractive");
+ expect(selected.reason).toBe("table_or_threshold_source_extractive");
+ });
+
+ it("keeps broad summaries on the fast synthesis path", () => {
+ const selected = route("Summarize the admission information guidance", [source()]);
+
+ expect(selected.mode).toBe("fast");
+ expect(selected.reason).toBe("strong_routine_retrieval");
});
it("uses the strong model for explicit multi-document comparisons", () => {
@@ -98,7 +135,28 @@ describe("RAG answer routing", () => {
]);
expect(selected.mode).toBe("strong");
- expect(selected.reason).toBe("clinical_risk_or_complex_query");
+ expect(selected.reason).toBe("multi_document_comparison_synthesis");
+ });
+
+ it("uses the fast model for routine balanced multi-document synthesis", () => {
+ const selected = route("Summarize monitoring issues across these documents", [
+ source({ id: "chunk-1", document_id: "doc-1", title: "Lithium" }),
+ source({ id: "chunk-2", document_id: "doc-2", title: "Clozapine" }),
+ ]);
+
+ expect(selected.mode).toBe("fast");
+ expect(selected.model).toBe("fast-model");
+ expect(selected.reason).toBe("balanced_multi_document_synthesis");
+ });
+
+ it("uses fast synthesis for simple two-document comparisons with strong support", () => {
+ const selected = route("Compare admission and discharge requirements", [
+ source({ id: "chunk-1", document_id: "doc-1", title: "Admission" }),
+ source({ id: "chunk-2", document_id: "doc-2", title: "Discharge" }),
+ ]);
+
+ expect(selected.mode).toBe("fast");
+ expect(selected.reason).toBe("balanced_multi_document_synthesis");
});
it("skips generation when retrieval has no plausible support", () => {
@@ -110,6 +168,23 @@ describe("RAG answer routing", () => {
expect(selected.model).toBeNull();
});
+ it("skips generation for weak off-topic medication dose retrieval", () => {
+ const selected = route("What antibiotic dose is recommended for community-acquired pneumonia?", [
+ source({
+ title: "Agitation and Arousal Pharmacological Management",
+ file_name: "MHSP.AgitationArousalPharmaMgt.pdf",
+ content: "Agitation dose guidance for mental health inpatients.",
+ similarity: 0.35,
+ hybrid_score: 0.36,
+ text_rank: 0,
+ }),
+ ]);
+
+ expect(selected.mode).toBe("unsupported");
+ expect(selected.reason).toBe("weak_complex_query_support");
+ expect(selected.model).toBeNull();
+ });
+
it("retries a fast unsupported answer with the strong model when source hits are solid", () => {
const selected = route("How is admission information handled?", [
source(),
diff --git a/tests/rag-trust.test.ts b/tests/rag-trust.test.ts
index 2c73d212b..d0f749c13 100644
--- a/tests/rag-trust.test.ts
+++ b/tests/rag-trust.test.ts
@@ -91,7 +91,83 @@ describe("RAG trust validation", () => {
title: "WA source",
});
expect(answer.citations[0].source_metadata?.document_status).toBe("current");
- expect(answer.answerSections).toHaveLength(1);
+ const sections = answer.answerSections ?? [];
+ expect(sections).toHaveLength(1);
+ });
+
+ it("removes JSON-like artifact sections while keeping valid sections", () => {
+ const answer = parseAnswerJson(
+ JSON.stringify({
+ answer: "Supported by source.",
+ grounded: true,
+ confidence: "medium",
+ citations: [{ chunk_id: "chunk-1" }],
+ answerSections: [
+ {
+ heading: "Monitoring",
+ body: "Monitor the patient closely for escalation.",
+ citation_chunk_ids: ["chunk-1"],
+ },
+ {
+ heading: `{\"answer\":\"or \",\"citation_chunk_ids\":[\"chunk-1\"]}`,
+ body: `{\"answer\":\" or \",\"citation_chunk_ids\":[\"chunk-1\"]}`,
+ citation_chunk_ids: ["chunk-1"],
+ },
+ ],
+ }),
+ [source()],
+ );
+
+ const sections = answer.answerSections ?? [];
+ expect(sections).toHaveLength(1);
+ expect(sections[0]?.heading).toBe("Monitoring");
+ expect(sections[0]?.body).toBe("Monitor the patient closely for escalation.");
+ });
+
+ it("returns empty answer sections when all sections are artifact-shaped", () => {
+ const answer = parseAnswerJson(
+ JSON.stringify({
+ answer: "Supported by source.",
+ grounded: true,
+ confidence: "medium",
+ citations: [{ chunk_id: "chunk-1" }],
+ answerSections: [
+ {
+ heading: `{\"answer\":\"or \",\"heading\":\"monitor\",\"citation_chunk_ids\":[\"chunk-1\"]}`,
+ body: `{\"answer\":\"Supported with JSON-shape\", \"citation_chunk_ids\":[\"chunk-1\"]}`,
+ citation_chunk_ids: ["chunk-1"],
+ },
+ ],
+ }),
+ [source()],
+ );
+
+ const sections = answer.answerSections ?? [];
+ expect(sections).toHaveLength(0);
+ });
+
+ it("keeps valid sections while filtering invalid citation IDs", () => {
+ const answer = parseAnswerJson(
+ JSON.stringify({
+ answer: "Supported and scoped.",
+ grounded: true,
+ confidence: "high",
+ citations: [{ chunk_id: "chunk-1" }],
+ answerSections: [
+ { heading: "Escalation", body: "Escalate on acute risk signs.", citation_chunk_ids: ["chunk-1", "missing"] },
+ {
+ heading: '{"heading":"artifact","body":"should be removed"}',
+ body: `{\"heading\":\"artifact\",\"body\":\"should be removed\"}`,
+ citation_chunk_ids: ["chunk-1"],
+ },
+ ],
+ }),
+ [source()],
+ );
+
+ const sections = answer.answerSections ?? [];
+ expect(sections).toHaveLength(1);
+ expect(sections[0]?.citation_chunk_ids).toEqual(["chunk-1"]);
});
it("includes exact citation chunk IDs in the model source block", () => {
@@ -101,6 +177,46 @@ describe("RAG trust validation", () => {
expect(block).toContain("document_id: doc-1");
});
+ it("does not pack administrative table images into model source context", () => {
+ const block = buildRagSourceBlock([
+ source({
+ images: [
+ {
+ id: "admin-table",
+ page_number: 3,
+ storage_path: "private/admin.png",
+ caption: "Authorisation and publication details.",
+ searchable: true,
+ image_type: "clinical_table",
+ source_kind: "table_crop",
+ tableRole: "admin",
+ tableTextSnippet: "Authorised by | Authorisation date | Published date",
+ metadata: {
+ clinical_use_class: "administrative",
+ table_role: "admin",
+ table_text: "Authorised by | Authorisation date | Published date",
+ },
+ },
+ ],
+ }),
+ ]);
+
+ expect(block).not.toContain("Authorisation date");
+ expect(block).not.toContain("Images:");
+ });
+
+ it("packs adjacent context under the same citation without creating new citation IDs", () => {
+ const block = buildRagSourceBlock([
+ source({
+ adjacent_context: "Previous and next table rows explain monitoring timing and escalation thresholds.",
+ }),
+ ]);
+
+ expect(block).toContain("Nearby context from the same source");
+ expect(block).toContain("monitoring timing and escalation thresholds");
+ expect(block).not.toContain("citation_chunk_id: Previous");
+ });
+
it("clamps model confidence to retrieval strength", () => {
const answer = parseAnswerJson(
JSON.stringify({
diff --git a/tests/smart-rag-api.test.ts b/tests/smart-rag-api.test.ts
new file mode 100644
index 000000000..e8b83355f
--- /dev/null
+++ b/tests/smart-rag-api.test.ts
@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest";
+import { buildSmartRagApiPlan } from "../src/lib/smart-rag-api";
+import type { SearchResult } from "../src/lib/types";
+
+function source(overrides: Partial = {}): SearchResult {
+ return {
+ id: overrides.id ?? "chunk-1",
+ document_id: overrides.document_id ?? "doc-1",
+ title: overrides.title ?? "Clinical Guideline",
+ file_name: overrides.file_name ?? "guideline.pdf",
+ page_number: overrides.page_number ?? 3,
+ chunk_index: overrides.chunk_index ?? 1,
+ section_heading: overrides.section_heading ?? "Monitoring",
+ content: overrides.content ?? "Monitor observations and escalate urgent risk.",
+ image_ids: [],
+ similarity: overrides.similarity ?? 0.86,
+ hybrid_score: overrides.hybrid_score ?? 0.92,
+ images: [],
+ ...overrides,
+ };
+}
+
+describe("smart RAG API plan", () => {
+ it("builds clickable core source links for answer responses", () => {
+ const plan = buildSmartRagApiPlan({
+ query: "What monitoring escalation is required?",
+ queryClass: "medication_dose_risk",
+ results: [source({ id: "chunk-a", document_id: "doc-a", title: "Monitoring Guide", page_number: 7 })],
+ routeMode: "fast",
+ retrievalStrategy: "hybrid",
+ });
+
+ expect(plan.intent).toBe("medication_or_risk_answer");
+ expect(plan.responseMode).toBe("fast_grounded_answer");
+ expect(plan.answerFocus).toContain("medication");
+ expect(plan.coreSourceLinks).toHaveLength(1);
+ expect(plan.coreSourceLinks[0]).toMatchObject({
+ href: "/documents/doc-a?page=7&chunk=chunk-a",
+ reason: "Medication, dose, monitoring, or risk evidence",
+ });
+ });
+
+ it("plans multi-document synthesis when the question asks to combine sources", () => {
+ const plan = buildSmartRagApiPlan({
+ query: "Combine monitoring guidance across documents",
+ queryClass: "broad_summary",
+ results: [
+ source({ id: "chunk-a", document_id: "doc-a", title: "Lithium", hybrid_score: 0.9 }),
+ source({ id: "chunk-b", document_id: "doc-b", title: "Clozapine", hybrid_score: 0.82 }),
+ ],
+ routeMode: "fast",
+ retrievalStrategy: "text_fast_path",
+ });
+
+ expect(plan.responseMode).toBe("multi_document_synthesis");
+ expect(plan.latencyPlan).toBe("cache_or_text_first");
+ expect(plan.answerFocus).toContain("2 documents");
+ expect(plan.streamPlan).toContain("Fuse strongest points");
+ expect(plan.coreSourceLinks.map((link) => link.document_id)).toEqual(["doc-a", "doc-b"]);
+ });
+
+ it("can be forced into document lookup mode for document-search API calls", () => {
+ const plan = buildSmartRagApiPlan({
+ query: "agitation guideline",
+ queryClass: "unsupported_or_general",
+ results: [source({ title: "Agitation Guideline" })],
+ retrievalStrategy: "document_lookup_fast_path",
+ preferredResponseMode: "document_lookup",
+ });
+
+ expect(plan.responseMode).toBe("document_lookup");
+ expect(plan.answerFocus).toContain("best matching document");
+ });
+});
diff --git a/tests/source-text-sanitizer.test.ts b/tests/source-text-sanitizer.test.ts
new file mode 100644
index 000000000..f30cc9d64
--- /dev/null
+++ b/tests/source-text-sanitizer.test.ts
@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import {
+ cleanClinicalSummaryText,
+ sourceTextForDisplay,
+ sourceTextForDocumentViewer,
+ sourceTextForIndexedPage,
+ sourceTextForModel,
+} from "../src/lib/source-text-sanitizer";
+
+describe("source text sanitizer", () => {
+ it("removes complete and partial internal image metadata from display text", () => {
+ const text =
+ "Source mentions: [[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Image type: clinical_table; Table text: | Dose | Route | [[IMAGE_DATA_END]] Continue oral medication when indicated.";
+
+ const display = sourceTextForDisplay(text);
+
+ expect(display).toBe("Source mentions: Continue oral medication when indicated.");
+ expect(display).not.toContain("[[IMAGE_DATA_START]]");
+ expect(display).not.toContain("Image ID:");
+ expect(display).not.toContain("Table text:");
+ });
+
+ it("keeps readable image descriptions for model context without internal fields", () => {
+ const text =
+ "[[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Table title: Agitation dose table; Table text: | Dose | Route |; Description: Oral and IM medication options. [[IMAGE_DATA_END]]";
+
+ const modelText = sourceTextForModel(text);
+
+ expect(modelText).toContain("Clinical table: Agitation dose table");
+ expect(modelText).toContain("Oral and IM medication options");
+ expect(modelText).not.toContain("Image ID:");
+ expect(modelText).not.toContain("Source kind:");
+ expect(modelText).not.toContain("[[IMAGE_DATA_START]]");
+ });
+
+ it("converts embedded table metadata into readable document-viewer text", () => {
+ const text =
+ "[[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Image type: clinical_table; Table role: clinical; Table text: | Time since last Clozapine dose | Clozapine dose | Blood test monitoring || --- | --- | --- || <48 hours | Restart at normal dose of Clozapine | No changes to monitoring || ≥48 hours - ≤72 hours | Restart Clozapine at 12.5mg | No changes to monitoring |; Description: Table showing Clozapine dose adjustment and blood test monitoring protocol. [[IMAGE_DATA_END]]";
+
+ const viewerText = sourceTextForDocumentViewer(text);
+
+ expect(viewerText).toContain("Table showing Clozapine dose adjustment");
+ expect(viewerText).toContain("Time since last Clozapine dose | Clozapine dose | Blood test monitoring");
+ expect(viewerText).toContain("- <48 hours: Clozapine dose: Restart at normal dose of Clozapine; Blood test monitoring: No changes to monitoring");
+ expect(viewerText).not.toContain("[[IMAGE_DATA_START]]");
+ expect(viewerText).not.toContain("Image ID:");
+ expect(viewerText).not.toContain("Source kind:");
+ expect(viewerText).not.toContain("Table text:");
+ });
+
+ it("cleans generated clinical summaries while preserving safe markdown bold", () => {
+ const summary =
+ "Key practical points: **clozapine** monitoring is required. Source mentions: [[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Table text: | Dose | [[IMAGE_DATA_END]]";
+
+ const cleaned = cleanClinicalSummaryText(summary);
+
+ expect(cleaned).toBe("**clozapine** monitoring is required.");
+ expect(cleaned).not.toContain("Key practical points:");
+ expect(cleaned).not.toContain("Source mentions:");
+ expect(cleaned).not.toContain("[[IMAGE_DATA_START]]");
+ expect(cleaned).not.toContain("Image ID:");
+ });
+
+ it("does not strip leading safe-bold markers during repeated summary cleaning", () => {
+ const once = cleanClinicalSummaryText("Key practical points: **clozapine** monitoring is required.");
+
+ expect(cleanClinicalSummaryText(once)).toBe("**clozapine** monitoring is required.");
+ });
+
+ it("preserves fixed-width indexed page spacing for table parsing", () => {
+ const text =
+ " Time since last Clozapine Clozapine dose Blood test monitoring\n dose\n\n <48 hours Restart at normal dose of No changes to monitoring";
+
+ const cleaned = sourceTextForIndexedPage(text);
+
+ expect(cleaned).toContain("Clozapine Clozapine dose");
+ expect(cleaned).toContain("dose Blood test monitoring");
+ });
+});
diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts
index c6c876116..492860ba7 100644
--- a/tests/supabase-schema.test.ts
+++ b/tests/supabase-schema.test.ts
@@ -8,15 +8,27 @@ describe("Supabase schema Data API grants", () => {
expect(schema).toContain("public.import_batches,");
expect(schema).toContain("public.document_labels,");
expect(schema).toContain("public.document_summaries,");
- expect(schema).toMatch(/grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries to service_role;/);
+ expect(schema).toContain("public.storage_cleanup_jobs");
+ expect(schema).toMatch(
+ /grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.image_caption_cache, .*public\.document_labels, .*public\.document_summaries, .*public\.document_sections, .*public\.document_memory_cards, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to service_role;/,
+ );
expect(schema).toContain("grant execute on all functions in schema public to service_role;");
});
- it("keeps authenticated direct access aligned with RLS policies", () => {
- expect(schema).toContain("grant select, insert, update, delete on table public.documents to authenticated;");
- expect(schema).toMatch(/grant select on table .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs to authenticated;/);
- expect(schema).toContain("grant select, insert on table public.rag_queries to authenticated;");
- expect(schema).not.toMatch(/grant .* on table .* to anon;/);
+ it("keeps browser Data API grants read-only except manual labels", () => {
+ expect(schema).toContain("revoke all privileges on all tables in schema public from anon, authenticated;");
+ expect(schema).toContain("revoke execute on all functions in schema public from public, anon, authenticated;");
+ expect(schema).toMatch(
+ /grant select on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to authenticated;/,
+ );
+ expect(schema).not.toContain("grant select, insert, update, delete on table public.documents to authenticated;");
+ expect(schema).not.toContain("grant select, insert on table public.rag_queries to authenticated;");
+ const authenticatedSelectGrant = schema.match(/grant select on table ([^;]+) to authenticated;/)?.[1] ?? "";
+ expect(authenticatedSelectGrant).not.toContain("public.document_sections");
+ expect(authenticatedSelectGrant).not.toContain("public.document_memory_cards");
+ expect(schema).not.toMatch(/grant [^;]* on table [^;]*public\.document_sections[^;]* to authenticated;/);
+ expect(schema).not.toMatch(/grant [^;]* on table [^;]*public\.document_memory_cards[^;]* to authenticated;/);
+ expect(schema).not.toMatch(/grant [^;]* on table [^;]* to anon;/);
});
it("supports bulk import queue claiming and reindex resets", () => {
@@ -28,14 +40,41 @@ describe("Supabase schema Data API grants", () => {
expect(schema).toContain("create or replace function public.claim_ingestion_jobs");
expect(schema).toContain("for update of j skip locked");
expect(schema).toContain("create or replace function public.reset_document_index");
+ expect(schema).toContain("delete from public.document_memory_cards where document_id = p_document_id;");
+ expect(schema).toContain("delete from public.document_sections where document_id = p_document_id;");
+ });
+
+ it("stores deep structured memory privately for source-backed answers", () => {
+ expect(schema).toContain("create table if not exists public.document_sections");
+ expect(schema).toContain("create table if not exists public.document_memory_cards");
+ expect(schema).toContain("card_type text not null");
+ expect(schema).toContain("source_chunk_ids uuid[] not null default '{}'");
+ expect(schema).toContain("create index if not exists document_memory_cards_search_idx");
+ expect(schema).toContain("create index if not exists document_memory_cards_embedding_hnsw_idx");
+ expect(schema).toContain("create or replace function public.stamp_document_deep_memory_version");
+ expect(schema).toContain("alter table public.document_sections enable row level security");
+ expect(schema).toContain("alter table public.document_memory_cards enable row level security");
+ });
+
+ it("tracks retryable storage cleanup and query-log purge performance", () => {
+ expect(schema).toContain("create table if not exists public.storage_cleanup_jobs");
+ expect(schema).toContain("create index if not exists storage_cleanup_jobs_owner_status_idx");
+ expect(schema).toContain("create index if not exists rag_queries_source_chunk_ids_gin_idx");
+ expect(schema).toContain('create policy "storage cleanup owner read"');
});
it("filters hybrid retrieval by owner inside Postgres", () => {
expect(schema).toContain("owner_filter uuid default null");
expect(schema).toContain("and (owner_filter is null or d.owner_id = owner_filter)");
expect(schema).toContain("create or replace function public.match_document_chunks_text");
+ expect(schema).toContain("create or replace function public.match_document_chunks_hybrid");
+ expect(schema).toContain("rrf_score double precision");
+ expect(schema).toContain("create or replace function public.match_document_memory_cards_hybrid");
+ expect(schema).toContain("create or replace function public.match_documents_for_query");
expect(schema).toContain("c.search_tsv @@ query.tsq");
- expect(schema).toContain("create index if not exists documents_search_idx on public.documents using gin(search_tsv)");
+ expect(schema).toContain(
+ "create index if not exists documents_search_idx on public.documents using gin(search_tsv)",
+ );
expect(schema).toContain("ts_rank_cd(d.title_search_tsv, query.tsq) * 3.0");
});
@@ -45,12 +84,19 @@ describe("Supabase schema Data API grants", () => {
expect(schema).toContain("clinical_relevance_score real not null default 0");
expect(schema).toContain("create table if not exists public.document_labels");
expect(schema).toContain("create table if not exists public.document_summaries");
- expect(schema).toContain("create policy \"labels owner manual insert\"");
- expect(schema).toContain("create policy \"summaries owner read\"");
+ expect(schema).toContain('create policy "labels owner manual insert"');
+ expect(schema).toContain('create policy "summaries owner read"');
expect(schema).toContain("create or replace function public.chunk_image_metadata");
expect(schema).toContain("and i.searchable = true");
expect(schema).toContain("and i.image_type <> 'logo_decorative'");
expect(schema).toContain("'clinical_relevance_score', i.clinical_relevance_score");
+ expect(schema).toContain("'sourceKind', i.source_kind");
+ expect(schema).toContain("'tableLabel', nullif(i.metadata->>'table_label', '')");
+ expect(schema).toContain("'tableTitle', nullif(i.metadata->>'table_title', '')");
+ expect(schema).toContain("'tableRole', nullif(i.metadata->>'table_role', '')");
+ expect(schema).toContain(
+ "'tableTextSnippet', nullif(left(coalesce(i.metadata->>'table_text_snippet', i.metadata->>'table_text', ''), 500), '')",
+ );
expect(schema).toContain("create or replace function public.get_related_document_metadata");
});
});
diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts
index 2ceaa904e..c382f490f 100644
--- a/tests/ui-smoke.spec.ts
+++ b/tests/ui-smoke.spec.ts
@@ -39,6 +39,17 @@ async function mockPrivateUnauthenticatedApi(page: Page) {
json: { demoMode: false, checks: readySetupChecks },
});
});
+ await page.route(/\/api\/answer(?:\/stream)?(?:\?.*)?$/, async (route) => {
+ const body = route.request().postDataJSON() as {
+ query?: string;
+ documentId?: string;
+ documentIds?: string[];
+ };
+ await fulfillAnswerResponse(
+ route,
+ demoAnswer(body.query ?? "What monitoring is required?", body.documentId, body.documentIds),
+ );
+ });
}
async function seedAuthenticatedSession(page: Page) {
@@ -72,8 +83,13 @@ async function mockPrivateAuthenticatedApi(page: Page) {
json: { demoMode: false, checks: readySetupChecks },
});
});
- await page.route(/\/api\/documents$/, async (route) => {
- await route.fulfill({ json: { documents: [] } });
+ await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => {
+ await route.fulfill({
+ json: {
+ documents: [],
+ pagination: { limit: 150, offset: 0, total: 0, nextOffset: 0, hasMore: false },
+ },
+ });
});
await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => {
await route.fulfill({ json: { jobs: [] } });
@@ -148,8 +164,20 @@ async function mockDemoApi(page: Page) {
json: { demoMode: true, checks: readySetupChecks },
});
});
- await page.route(/\/api\/documents$/, async (route) => {
- await route.fulfill({ json: { documents: demoDocuments, demoMode: true } });
+ await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => {
+ await route.fulfill({
+ json: {
+ documents: demoDocuments,
+ demoMode: true,
+ pagination: {
+ limit: 150,
+ offset: 0,
+ total: demoDocuments.length,
+ nextOffset: demoDocuments.length,
+ hasMore: false,
+ },
+ },
+ });
});
await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => {
await route.fulfill({ json: { jobs: [], demoMode: true } });
@@ -168,6 +196,47 @@ async function mockDemoApi(page: Page) {
demoMode: true,
});
});
+ await page.route(/\/api\/search$/, async (route) => {
+ await route.fulfill({
+ json: {
+ results: [
+ {
+ id: "44444444-4444-4444-8444-444444444442",
+ document_id: "11111111-1111-4111-8111-111111111111",
+ title: "Synthetic lithium monitoring protocol",
+ file_name: "lithium-monitoring.pdf",
+ page_number: 1,
+ chunk_index: 0,
+ section_heading: "Monitoring",
+ content: "Lithium monitoring and toxicity safety-net source passage.",
+ image_ids: [],
+ similarity: 0.9,
+ hybrid_score: 0.92,
+ images: [],
+ },
+ ],
+ visualEvidence: [],
+ relatedDocuments: [],
+ documentMatches: [
+ {
+ document_id: "11111111-1111-4111-8111-111111111111",
+ title: "Synthetic lithium monitoring protocol",
+ file_name: "lithium-monitoring.pdf",
+ labels: [{ label: "lithium", label_type: "medication", source: "generated", confidence: 0.94 }],
+ summarySnippet: "Lithium monitoring and toxicity safety-net reminders.",
+ bestPages: [1],
+ bestChunkIds: ["44444444-4444-4444-8444-444444444442"],
+ imageCount: 1,
+ tableCount: 1,
+ matchReason: "Matched indexed passage",
+ score: 0.92,
+ },
+ ],
+ smartPanel: {},
+ demoMode: true,
+ },
+ });
+ });
await page.route(/\/api\/documents\/([^/]+)\/signed-url(?:\?.*)?$/, async (route) => {
const id = new URL(route.request().url()).pathname.split("/").at(-2) ?? "";
const document = getDemoDocument(id);
@@ -179,15 +248,37 @@ async function mockDemoApi(page: Page) {
json: { url: document.storage_path, fileType: document.file_type, demoMode: true },
});
});
+ await page.route(/\/api\/documents\/[^/]+\/summarize$/, async (route) => {
+ await route.fulfill({
+ json: {
+ answer:
+ "Key practical points: **clozapine** monitoring requires regular FBC/ANC checks and review of constipation, myocarditis symptoms, metabolic risk, and missed-dose restart rules.",
+ grounded: true,
+ confidence: "high",
+ citations: [],
+ sources: [],
+ demoMode: true,
+ },
+ });
+ });
await page.route(/\/api\/documents\/([^/]+)(?:\?.*)?$/, async (route) => {
const url = new URL(route.request().url());
const id = url.pathname.split("/").at(-1) ?? "";
- const payload = getDemoDocumentPayload(id, url.searchParams.get("chunk"));
+ const selectedChunkId = url.searchParams.get("chunk");
+ const payload = getDemoDocumentPayload(id, selectedChunkId);
if (!payload) {
await route.fulfill({ status: 404, json: { error: "Demo document not found." } });
return;
}
- await route.fulfill({ json: { ...payload, demoMode: true } });
+ const longSelectedPassage = selectedChunkId
+ ? {
+ ...payload,
+ chunks: payload.chunks.map((chunk) =>
+ chunk.id === selectedChunkId ? { ...chunk, content: Array(8).fill(chunk.content).join(" ") } : chunk,
+ ),
+ }
+ : payload;
+ await route.fulfill({ json: { ...longSelectedPassage, demoMode: true } });
});
}
@@ -198,9 +289,7 @@ async function expectDomIntegrity(page: Page, options: { mobileNav?: boolean } =
.filter((id, index, all) => id && all.indexOf(id) !== index);
const brokenAriaRefs: Array<{ attr: string; id: string }> = [];
- for (const element of [
- ...document.querySelectorAll("[aria-labelledby],[aria-describedby],[aria-controls]"),
- ]) {
+ for (const element of [...document.querySelectorAll("[aria-labelledby],[aria-describedby],[aria-controls]")]) {
for (const attr of ["aria-labelledby", "aria-describedby", "aria-controls"]) {
const value = element.getAttribute(attr);
if (!value) continue;
@@ -214,7 +303,9 @@ async function expectDomIntegrity(page: Page, options: { mobileNav?: boolean } =
h1Count: document.querySelectorAll("h1,[role='heading'][aria-level='1']").length,
duplicateIds: [...new Set(duplicateIds)],
brokenAriaRefs,
- hasFrameworkOverlay: /Unhandled Runtime Error|Build Error|Application error|Next\.js/.test(document.body.innerText),
+ hasFrameworkOverlay: /Unhandled Runtime Error|Build Error|Application error|Next\.js/.test(
+ document.body.innerText,
+ ),
};
});
@@ -229,8 +320,8 @@ async function expectDomIntegrity(page: Page, options: { mobileNav?: boolean } =
}
async function waitForDemoDashboardReady(page: Page) {
- await expect(page.getByLabel("Ask a question across indexed guidelines")).toBeEnabled();
- await expect(page.getByText("3 documents")).toBeAttached({ timeout: 30000 });
+ await expect(page.getByLabel("Search indexed guidelines by question or keyword")).toBeEnabled();
+ await expect(page.getByTestId("scope-prompts-drawer").getByText("3 documents")).toBeAttached({ timeout: 30000 });
}
async function scrollDashboardToBottom(page: Page) {
@@ -278,19 +369,29 @@ test.describe("Clinical KB UI smoke coverage", () => {
await expect(page.getByRole("heading", { level: 1, name: "Clinical Guide" })).toBeVisible();
await expect(page.getByRole("heading", { name: "Answer" })).toBeVisible();
- await expect(page.getByLabel("Ask a question across indexed guidelines")).toBeVisible();
+ await expect(page.getByLabel("Search indexed guidelines by question or keyword")).toBeVisible();
+ if (viewport.width >= 640) {
+ const scopeDrawer = page.getByTestId("scope-prompts-drawer");
+ await expect(scopeDrawer).toBeVisible();
+ expect(await scopeDrawer.evaluate((element) => element.hasAttribute("open"))).toBe(false);
+ await expect(scopeDrawer).toContainText("Scope & prompts");
+ }
await expectDomIntegrity(page, { mobileNav: viewport.width < 1024 });
await expectNoPageHorizontalOverflow(page);
});
}
- test("private mode unauthenticated dashboard keeps search disabled and DOM coherent", async ({ page }) => {
+ test("private mode unauthenticated dashboard allows public search", async ({ page }) => {
await page.setViewportSize({ width: 390, height: 820 });
await mockPrivateUnauthenticatedApi(page);
await gotoApp(page, "/");
- await expect(page.getByText("Sign in for private documents")).toBeVisible();
- await expect(page.getByRole("button", { name: "Ask" })).toBeDisabled();
+ const questionInput = page.getByLabel("Search indexed guidelines by question or keyword");
+ await questionInput.fill("lithium monitoring");
+ await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeEnabled();
+ await page.getByRole("button", { name: "Generate source-backed answer" }).click();
+ await expect(page.getByTestId("answer-grounding-chip")).toBeVisible();
+ await expect(page.getByText("Sign in before searching private guideline documents")).toHaveCount(0);
await expect(page.getByRole("heading", { level: 1, name: "Clinical Guide" })).toBeVisible();
await expectDomIntegrity(page, { mobileNav: true });
await expectNoPageHorizontalOverflow(page);
@@ -302,20 +403,26 @@ test.describe("Clinical KB UI smoke coverage", () => {
await gotoApp(page, "/");
await waitForDemoDashboardReady(page);
- const questionInput = page.getByLabel("Ask a question across indexed guidelines");
+ const questionInput = page.getByLabel("Search indexed guidelines by question or keyword");
await expect(questionInput).toBeEnabled();
await expect(page.getByLabel("Open document scope and prompt controls")).toBeVisible();
const question = "What toxicity safety-net symptoms should be reviewed for lithium?";
await questionInput.click();
await questionInput.pressSequentially(question);
await expect(questionInput).toHaveValue(question);
- await expect(page.getByRole("button", { name: "Ask" })).toBeEnabled();
- await page.getByRole("button", { name: "Ask" }).click();
+ await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeEnabled();
+ await page.getByRole("button", { name: "Generate source-backed answer" }).click();
- await expect(page.getByLabel("Source-backed answer")).toBeVisible();
+ await expect(page.getByTestId("answer-grounding-chip")).toBeVisible();
+ await expect(page.getByTestId("clinical-action-view")).toBeVisible();
+ await expect(page.getByTestId("clinical-action-view").getByRole("heading", { name: "Action", exact: true })).toBeVisible();
+ await expect(page.getByTestId("clinical-action-view").getByRole("heading", { name: "Thresholds", exact: true })).toBeVisible();
+ const rawNarrative = page.getByTestId("raw-answer-narrative");
+ await expect(rawNarrative).toBeVisible();
+ expect(await rawNarrative.evaluate((element) => element.hasAttribute("open"))).toBe(false);
await expect(page.getByTestId("answer-top-source-chip")).toBeVisible();
await expect(page.getByTestId("answer-grounding-chip")).toBeVisible();
- await expect(page.getByText(/Synthetic demo only/i)).toBeVisible();
+ await expect(page.getByTestId("clinical-action-view").getByText(/Synthetic demo only/i)).toBeVisible();
await expect(
page.getByText("Draft only; verify source first before pasting into the medical record.").first(),
).toBeVisible();
@@ -356,7 +463,7 @@ test.describe("Clinical KB UI smoke coverage", () => {
expect(headingMetrics.actionHeight).toBeLessThanOrEqual(34);
await expectMobileNavTarget(page, "Quotes", "#quotes", "Source quotes");
- await expectMobileNavTarget(page, "Images", "#images", "Source diagrams");
+ await expectMobileNavTarget(page, "Images", "#images", "Tables and diagrams");
await expectMobileNavTarget(page, "Sources", "#sources", "Source passages");
await page.getByLabel("Open document scope and prompt controls").click();
@@ -377,6 +484,29 @@ test.describe("Clinical KB UI smoke coverage", () => {
await expectNoPageHorizontalOverflow(page);
});
+ test("document search mode lists matching documents and scope actions", async ({ page }) => {
+ await page.setViewportSize({ width: 390, height: 820 });
+ await mockDemoApi(page);
+ await gotoApp(page, "/");
+ await waitForDemoDashboardReady(page);
+
+ await page.getByRole("button", { name: "Switch to document search mode" }).click();
+ await expect(page.getByRole("heading", { name: "Document matches" })).toBeVisible();
+ await expect(page.getByRole("button", { name: "Find matching documents" })).toBeDisabled();
+ await expect(page.getByText("Search documents")).toBeVisible();
+
+ const questionInput = page.getByLabel("Search indexed guidelines by question or keyword");
+ await questionInput.fill("lithium monitoring");
+ await page.getByRole("button", { name: "Find matching documents" }).click();
+
+ await expect(page.getByText("Synthetic lithium monitoring protocol").first()).toBeVisible();
+ await expect(page.getByText("1 tables")).toBeVisible();
+ await expect(page.getByRole("button", { name: /Scope search to/i }).first()).toBeVisible();
+ await page.getByRole("button", { name: /Answer from/i }).first().click();
+ await expect(page.getByRole("heading", { name: "Answer" })).toBeVisible();
+ await expectNoPageHorizontalOverflow(page);
+ });
+
test("document viewer puts pinned evidence before the PDF preview on mobile", async ({ page }) => {
await page.setViewportSize({ width: 320, height: 720 });
await mockDemoApi(page);
@@ -389,8 +519,11 @@ test.describe("Clinical KB UI smoke coverage", () => {
const preview = page.getByTestId("pdf-preview");
const toolbar = page.getByTestId("pdf-toolbar");
const pdfScroller = page.getByTestId("pdf-canvas-scroll");
+ const viewerNav = page.getByRole("navigation", { name: "Document viewer sections" }).first();
await expect(evidence).toBeVisible();
+ await expect(viewerNav.getByRole("link", { name: "Evidence" })).toBeVisible();
+ await expect(viewerNav.getByRole("link", { name: "PDF" })).toBeVisible();
await expect(page.getByRole("heading", { level: 1, name: "Synthetic lithium monitoring protocol" })).toBeVisible();
await expect(preview).toBeVisible();
await expect(toolbar).toBeVisible({ timeout: 30000 });
@@ -398,23 +531,45 @@ test.describe("Clinical KB UI smoke coverage", () => {
const evidenceBox = await evidence.boundingBox();
const previewBox = await preview.boundingBox();
- const indexedTextBox = await page.getByText("Indexed page text").boundingBox();
- const imagesBox = await page.getByText("Images and captions").boundingBox();
+ const indexedTextBox = await page.getByText("Indexed page text", { exact: true }).boundingBox();
+ const imagesBox = await page.getByRole("heading", { name: "Tables and diagrams" }).boundingBox();
expect(evidenceBox).not.toBeNull();
expect(previewBox).not.toBeNull();
expect(indexedTextBox).not.toBeNull();
expect(imagesBox).not.toBeNull();
expect(evidenceBox!.y).toBeLessThan(previewBox!.y);
+ expect(evidenceBox!.height).toBeLessThan(640);
expect(indexedTextBox!.y).toBeLessThan(previewBox!.y);
expect(indexedTextBox!.y).toBeLessThan(imagesBox!.y);
+ const passageToggle = page.getByTestId("toggle-full-passage").first();
+ await expect(passageToggle).toHaveText("Show full passage");
+ await passageToggle.click();
+ await expect(passageToggle).toHaveText("Show passage preview");
+ const expandedEvidenceBox = await evidence.boundingBox();
+ expect(expandedEvidenceBox?.height ?? 0).toBeGreaterThan(evidenceBox!.height);
+ await viewerNav.getByRole("link", { name: "PDF" }).click();
+ await expect(preview).toBeInViewport();
+
const mobilePdfStyles = await toolbar.evaluate((element) => ({
position: window.getComputedStyle(element).position,
}));
expect(mobilePdfStyles.position).toBe("static");
await expect(pdfScroller).toBeVisible();
+ await page.getByRole("button", { name: "Fit page width and enter fullscreen" }).click();
+ await expect(page.getByRole("button", { name: "Exit fullscreen document view" })).toBeVisible();
+ const fullscreenRootStyles = await page.getByTestId("pdf-fullscreen-root").evaluate((element) => {
+ const style = window.getComputedStyle(element);
+ return {
+ position: style.position,
+ height: style.height,
+ };
+ });
+ expect(fullscreenRootStyles.position).toBe("fixed");
+ await page.getByRole("button", { name: "Exit fullscreen document view" }).click();
+
const fitWidthScrollStyles = await pdfScroller.evaluate((element) => {
const style = window.getComputedStyle(element);
return {
@@ -427,6 +582,31 @@ test.describe("Clinical KB UI smoke coverage", () => {
await expectNoPageHorizontalOverflow(page);
});
+ test("document summary opens at the top with cleaned bold formatting", async ({ page }) => {
+ await page.setViewportSize({ width: 390, height: 820 });
+ await mockDemoApi(page);
+ await gotoApp(
+ page,
+ "/documents/11111111-1111-4111-8111-111111111111?page=1&chunk=44444444-4444-4444-8444-444444444442",
+ );
+
+ await page.getByRole("button", { name: "Summarise document" }).click();
+
+ const generatedSummary = page.getByTestId("generated-clinical-summary");
+ await expect(generatedSummary).toBeVisible();
+ await expect(generatedSummary).toContainText("clozapine monitoring requires regular FBC/ANC checks");
+ await expect(generatedSummary).not.toContainText("Key practical points:");
+ await expect(generatedSummary).not.toContainText("**");
+ await expect(generatedSummary.locator("strong").filter({ hasText: "clozapine" })).toHaveCount(1);
+
+ const summaryBox = await generatedSummary.boundingBox();
+ const previewBox = await page.getByTestId("pdf-preview").boundingBox();
+ expect(summaryBox).not.toBeNull();
+ expect(previewBox).not.toBeNull();
+ expect(summaryBox!.y).toBeLessThan(previewBox!.y);
+ await expectNoPageHorizontalOverflow(page);
+ });
+
test("document viewer failed preview exposes retry recovery", async ({ page }) => {
await page.route(/\/api\/setup-status$/, async (route) => {
await route.fulfill({ json: { demoMode: true, checks: readySetupChecks } });
@@ -498,7 +678,7 @@ test.describe("Clinical KB UI smoke coverage", () => {
await page.setViewportSize({ width: 414, height: 820 });
await mockPrivateUnauthenticatedApi(page);
await gotoApp(page, "/");
- await expect(page.getByLabel("Ask a question across indexed guidelines")).toBeVisible();
+ await expect(page.getByLabel("Search indexed guidelines by question or keyword")).toBeVisible();
await scrollDashboardToBottom(page);
const uploadSummary = page.locator("summary").filter({ hasText: "Upload and indexing" }).first();
diff --git a/tests/ward-output.test.ts b/tests/ward-output.test.ts
index 73af05182..401875d5d 100644
--- a/tests/ward-output.test.ts
+++ b/tests/ward-output.test.ts
@@ -55,12 +55,58 @@ describe("ward output helpers", () => {
const sections = buildClinicalOutputSections(answer);
expect(sections.map((section) => section.id)).toEqual([
- "key-actions",
- "monitoring-checklist",
- "escalation-triggers",
+ "action",
+ "monitoring",
+ "thresholds",
+ "escalation",
+ "verify-source",
]);
expect(sections[1].items[0]).toContain("renal function");
- expect(sections[2].items[0]).toContain("vomiting");
+ expect(sections[2].items[0]).toContain("lithium level");
+ expect(sections[3].items[0]).toContain("vomiting");
+ expect(sections[4].items[0]).toContain("1 linked citation");
+ });
+
+ it("places extracted threshold tables before threshold prose", () => {
+ const thresholdAnswer: RagAnswer = {
+ ...answer,
+ answer: "Withhold clozapine if ANC is below the required threshold and urgently review.",
+ answerSections: [
+ {
+ heading: "Threshold",
+ body: "Withhold clozapine if ANC is below the required threshold and urgently review.",
+ citation_chunk_ids: ["chunk-1"],
+ },
+ ],
+ visualEvidence: [
+ {
+ id: "image-1",
+ image_id: "image-1",
+ signed_url_endpoint: "/api/images/image-1/signed-url",
+ caption: "FBC/ANC monitoring thresholds",
+ document_id: "doc-1",
+ title: "Clozapine source",
+ file_name: "clozapine.pdf",
+ page_number: 2,
+ source_chunk_id: "chunk-1",
+ chunk_index: 0,
+ viewer_href: "/documents/doc-1?page=2&chunk=chunk-1",
+ tableLabel: "Table 1",
+ tableTitle: "FBC/ANC thresholds",
+ tableRows: [
+ ["ANC", "Action"],
+ ["Below threshold", "Withhold clozapine and review"],
+ ],
+ tableColumns: ["Threshold", "Action"],
+ },
+ ],
+ };
+
+ const thresholds = buildClinicalOutputSections(thresholdAnswer).find((section) => section.id === "thresholds");
+
+ expect(thresholds?.tables).toHaveLength(1);
+ expect(thresholds?.tables?.[0].caption).toContain("FBC/ANC");
+ expect(thresholds?.items[0]).toContain("Withhold clozapine");
});
it("formats copyable answer and quote text with citations", () => {
@@ -82,6 +128,7 @@ describe("ward output helpers", () => {
it("pauses polling in demo mode or hidden tabs", () => {
expect(shouldPollForUpdates(false, "visible")).toBe(true);
+ expect(shouldPollForUpdates(false, "visible", false)).toBe(false);
expect(shouldPollForUpdates(true, "visible")).toBe(false);
expect(shouldPollForUpdates(false, "hidden")).toBe(false);
});
diff --git a/worker/main.ts b/worker/main.ts
index e6624f4f1..c26fc88c9 100644
--- a/worker/main.ts
+++ b/worker/main.ts
@@ -4,9 +4,16 @@ import os from "node:os";
import path from "node:path";
import { env } from "../src/lib/env";
import { buildChunks } from "../src/lib/chunking";
-import { upsertDocumentEnrichment } from "../src/lib/document-enrichment";
+import { ragEnrichmentVersion, upsertDocumentEnrichment } from "../src/lib/document-enrichment";
+import { ragDeepMemoryVersion, upsertDocumentDeepMemory } from "../src/lib/deep-memory";
import { extractDocument, fileToBase64 } from "../src/lib/extractors/document";
-import { cheapImageSkipReason, classifiedImageSkipReason, lightweightPerceptualHash } from "../src/lib/image-filtering";
+import {
+ assessClinicalImageUse,
+ cheapImageSkipReason,
+ classifiedImageSkipReason,
+ clinicalImagePolicyVersion,
+ lightweightPerceptualHash,
+} from "../src/lib/image-filtering";
import { isRetryableIngestionError, nextRetryAt, terminalBatchStatus } from "../src/lib/ingestion";
import { classifyAndCaptionImageFromBase64, embedTexts } from "../src/lib/openai";
import { safeErrorLogDetails, safeIngestionJobLog } from "../src/lib/privacy";
@@ -134,7 +141,75 @@ function cachedImageMetadata(value: unknown) {
function cachedImageLabels(labels: unknown) {
if (!Array.isArray(labels)) return [];
- return labels.map((label) => String(label).trim()).filter(Boolean).slice(0, 6);
+ return labels
+ .map((label) => String(label).trim())
+ .filter(Boolean)
+ .slice(0, 6);
+}
+
+function metadataString(metadata: Record | undefined, key: string) {
+ const value = metadata?.[key];
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function compactMetadataText(value: string | null, limit = 1200) {
+ if (!value) return null;
+ const compact = value.replace(/\s+/g, " ").trim();
+ if (!compact) return null;
+ return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact;
+}
+
+function imageTableMetadata(image: ExtractedDocument["images"][number]) {
+ const metadata = image.metadata ?? {};
+ const tableText = metadataString(metadata, "table_text");
+ const confidence = Number(metadata.table_confidence);
+ return {
+ candidateType: metadataString(metadata, "candidate_type"),
+ tableLabel: metadataString(metadata, "table_label"),
+ tableTitle: metadataString(metadata, "table_title"),
+ tableRole: metadataString(metadata, "table_role"),
+ tableConfidence: Number.isFinite(confidence) ? confidence : null,
+ tableText,
+ tableTextSnippet: compactMetadataText(tableText),
+ accessibleTableMarkdown: metadataString(metadata, "accessible_table_markdown") ?? tableText,
+ tableRows: Array.isArray(metadata.table_rows) ? metadata.table_rows : null,
+ tableColumns: Array.isArray(metadata.table_columns) ? metadata.table_columns : null,
+ };
+}
+
+function nonClinicalTableClassification(args: {
+ tableMetadata: ReturnType;
+ sourceKind?: string | null;
+ imageType?: ImageEvidenceCategory;
+}) {
+ const assessment = assessClinicalImageUse({
+ imageType: args.imageType ?? "clinical_table",
+ searchable: true,
+ clinicalRelevanceScore: 0.4,
+ sourceKind: args.sourceKind,
+ tableRole: args.tableMetadata.tableRole,
+ tableText: args.tableMetadata.tableText,
+ tableTitle: args.tableMetadata.tableTitle,
+ tableLabel: args.tableMetadata.tableLabel,
+ });
+ if (assessment.clinical_use_class === "clinical_evidence" || assessment.clinical_use_class === "ambiguous") {
+ return null;
+ }
+ return {
+ image_type: args.imageType ?? ("clinical_table" as const),
+ searchable: false,
+ clinical_relevance_score: 0,
+ labels: [],
+ caption:
+ assessment.clinical_use_class === "administrative"
+ ? "Administrative document-control table retained for audit, not clinical evidence."
+ : "Reference table retained for audit, not clinical evidence.",
+ skip_reason: assessment.clinical_use_reason,
+ clinical_use_class: assessment.clinical_use_class,
+ clinical_use_reason: assessment.clinical_use_reason,
+ clinical_signal_score: assessment.clinical_signal_score,
+ admin_signal_score: assessment.admin_signal_score,
+ } satisfies ImageClassification;
}
async function getCachedImageClassification(ownerId: string | null, imageHash: string) {
@@ -159,14 +234,28 @@ async function getCachedImageClassification(ownerId: string | null, imageHash: s
? (metadata.image_type as ImageEvidenceCategory)
: "unclear";
const score = Number(metadata.clinical_relevance_score);
+ const labels = cachedImageLabels(metadata.labels);
+ const assessment = assessClinicalImageUse({
+ imageType,
+ searchable: Boolean(metadata.searchable),
+ clinicalRelevanceScore: score,
+ caption: String(data.caption || ""),
+ labels,
+ skipReason: typeof metadata.skip_reason === "string" ? metadata.skip_reason : null,
+ });
return {
image_type: imageType,
- searchable: Boolean(metadata.searchable) && imageType !== "logo_decorative",
- clinical_relevance_score: Number.isFinite(score) ? Math.min(Math.max(score, 0), 1) : 0.4,
- labels: cachedImageLabels(metadata.labels),
+ searchable: assessment.searchable && imageType !== "logo_decorative",
+ clinical_relevance_score: assessment.clinical_relevance_score,
+ labels,
caption: String(data.caption || "").trim() || "Extracted source image.",
- skip_reason: typeof metadata.skip_reason === "string" && metadata.skip_reason.trim() ? metadata.skip_reason.trim() : null,
+ skip_reason:
+ typeof metadata.skip_reason === "string" && metadata.skip_reason.trim() ? metadata.skip_reason.trim() : null,
+ clinical_use_class: assessment.clinical_use_class,
+ clinical_use_reason: assessment.clinical_use_reason,
+ clinical_signal_score: assessment.clinical_signal_score,
+ admin_signal_score: assessment.admin_signal_score,
} satisfies ImageClassification;
}
@@ -192,6 +281,11 @@ async function setCachedImageClassification(args: {
clinical_relevance_score: args.classification.clinical_relevance_score,
labels: args.classification.labels,
skip_reason: args.classification.skip_reason,
+ clinical_use_class: args.classification.clinical_use_class,
+ clinical_use_reason: args.classification.clinical_use_reason,
+ clinical_signal_score: args.classification.clinical_signal_score,
+ admin_signal_score: args.classification.admin_signal_score,
+ image_policy_version: clinicalImagePolicyVersion,
},
updated_at: new Date().toISOString(),
},
@@ -209,7 +303,12 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument,
caption: string;
pageNumber: number | null;
imageType: ImageEvidenceCategory;
+ sourceKind: string | null;
labels: string[];
+ tableLabel: string | null;
+ tableTitle: string | null;
+ tableTextSnippet: string | null;
+ tableRole: string | null;
}> = [];
const seenHashes = new Set();
let skippedImages = 0;
@@ -239,13 +338,27 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument,
seenHashes.add(imageHash);
const nearbyText = image.pageNumber ? pagesByNumber.get(image.pageNumber) : undefined;
- let classification = await getCachedImageClassification(job.documents.owner_id, imageHash);
- const classificationCacheHit = Boolean(classification);
+ const tableMetadata = imageTableMetadata(image);
+ let classification: ImageClassification | null =
+ image.sourceKind === "table_crop"
+ ? nonClinicalTableClassification({ tableMetadata, sourceKind: image.sourceKind })
+ : null;
+ let classificationCacheHit = false;
+ if (!classification) {
+ classification = await getCachedImageClassification(job.documents.owner_id, imageHash);
+ classificationCacheHit = Boolean(classification);
+ }
if (!classification) {
classification = await classifyAndCaptionImageFromBase64({
base64: await fileToBase64(image.path),
mimeType: image.mimeType,
nearbyText,
+ sourceKind: image.sourceKind ?? null,
+ candidateType: tableMetadata.candidateType,
+ tableLabel: tableMetadata.tableLabel,
+ tableTitle: tableMetadata.tableTitle,
+ tableRole: tableMetadata.tableRole,
+ tableText: tableMetadata.tableText,
});
await setCachedImageClassification({
ownerId: job.documents.owner_id,
@@ -254,13 +367,44 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument,
classification,
});
}
+ const policyAssessment = assessClinicalImageUse({
+ imageType: classification.image_type,
+ searchable: classification.searchable,
+ clinicalRelevanceScore: classification.clinical_relevance_score,
+ sourceKind: image.sourceKind ?? null,
+ tableRole: tableMetadata.tableRole,
+ tableText: tableMetadata.tableText,
+ tableTitle: tableMetadata.tableTitle,
+ tableLabel: tableMetadata.tableLabel,
+ caption: classification.caption,
+ labels: classification.labels,
+ skipReason: classification.skip_reason,
+ });
+ classification = {
+ ...classification,
+ searchable: policyAssessment.searchable,
+ clinical_relevance_score: policyAssessment.clinical_relevance_score,
+ skip_reason: policyAssessment.searchable ? classification.skip_reason : policyAssessment.clinical_use_reason,
+ clinical_use_class: policyAssessment.clinical_use_class,
+ clinical_use_reason: policyAssessment.clinical_use_reason,
+ clinical_signal_score: policyAssessment.clinical_signal_score,
+ admin_signal_score: policyAssessment.admin_signal_score,
+ };
+
const classifiedSkipReason = classifiedImageSkipReason(classification);
- if (classifiedSkipReason) {
+ const retainAsAuditTable =
+ image.sourceKind === "table_crop" &&
+ ["administrative", "reference"].includes(policyAssessment.clinical_use_class) &&
+ classification.image_type !== "logo_decorative";
+ if (classifiedSkipReason && !retainAsAuditTable) {
skippedImages += 1;
skipReasons.set(classifiedSkipReason, (skipReasons.get(classifiedSkipReason) ?? 0) + 1);
continue;
}
- imageTypeCounts.set(classification.image_type, (imageTypeCounts.get(classification.image_type) ?? 0) + 1);
+ const persistedSearchable = policyAssessment.searchable;
+ if (persistedSearchable) {
+ imageTypeCounts.set(classification.image_type, (imageTypeCounts.get(classification.image_type) ?? 0) + 1);
+ }
const ext = path.extname(image.path) || ".png";
const imagePrefix = job.documents.owner_id
@@ -283,8 +427,8 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument,
caption: classification.caption,
bbox: image.bbox ?? null,
image_type: classification.image_type,
- searchable: classification.searchable,
- clinical_relevance_score: classification.clinical_relevance_score,
+ searchable: persistedSearchable,
+ clinical_relevance_score: persistedSearchable ? classification.clinical_relevance_score : 0,
source_kind: image.sourceKind ?? "embedded",
width: image.width ?? null,
height: image.height ?? null,
@@ -292,24 +436,49 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument,
perceptual_hash: perceptualHash,
labels: classification.labels,
metadata: {
+ ...(image.metadata ?? {}),
extractor: "local-worker",
image_hash: imageHash,
perceptual_hash: perceptualHash,
classification_cache_hit: classificationCacheHit,
- ...(image.metadata ?? {}),
+ candidate_type: tableMetadata.candidateType ?? image.metadata?.candidate_type ?? null,
+ table_label: tableMetadata.tableLabel,
+ table_title: tableMetadata.tableTitle,
+ table_text: tableMetadata.tableText,
+ table_text_snippet: tableMetadata.tableTextSnippet,
+ table_role: tableMetadata.tableRole,
+ table_confidence: tableMetadata.tableConfidence,
+ table_rows: tableMetadata.tableRows,
+ table_columns: tableMetadata.tableColumns,
+ accessible_table_markdown: tableMetadata.accessibleTableMarkdown,
+ clinical_use_class: policyAssessment.clinical_use_class,
+ clinical_use_reason: policyAssessment.clinical_use_reason,
+ clinical_signal_score: policyAssessment.clinical_signal_score,
+ admin_signal_score: policyAssessment.admin_signal_score,
+ image_policy_version: clinicalImagePolicyVersion,
+ retained_for_audit: retainAsAuditTable || undefined,
+ retained_for_document_view: retainAsAuditTable || undefined,
+ skip_reason: retainAsAuditTable ? classifiedSkipReason : classification.skip_reason,
},
})
- .select("id,caption,page_number,image_type,labels")
+ .select("id,caption,page_number,image_type,labels,searchable")
.single();
if (error) throw new Error(error.message);
- insertedImages.push({
- id: data.id,
- caption: data.caption,
- pageNumber: data.page_number,
- imageType: data.image_type,
- labels: data.labels ?? [],
- });
+ if (data.searchable !== false) {
+ insertedImages.push({
+ id: data.id,
+ caption: data.caption,
+ pageNumber: data.page_number,
+ imageType: data.image_type,
+ sourceKind: image.sourceKind ?? "embedded",
+ labels: data.labels ?? [],
+ tableLabel: tableMetadata.tableLabel,
+ tableTitle: tableMetadata.tableTitle,
+ tableTextSnippet: tableMetadata.tableTextSnippet,
+ tableRole: tableMetadata.tableRole,
+ });
+ }
}
return {
@@ -331,6 +500,8 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) {
content_hash: job.documents.content_hash ?? null,
embedding_model: env.OPENAI_EMBEDDING_MODEL,
extractor: "local-worker",
+ rag_indexing_version: ragDeepMemoryVersion,
+ rag_memory_version: ragDeepMemoryVersion,
};
const chunks = buildChunks(
extracted.pages.map((page) => ({
@@ -396,28 +567,35 @@ function extractionMetrics(
}
async function loadEnrichmentRows(documentId: string) {
- const [chunksResult, imagesResult] = await Promise.all([
- supabase
+ const chunks = [];
+ const images = [];
+
+ for (let start = 0; ; start += 1000) {
+ const { data, error } = await supabase
.from("document_chunks")
- .select("id,page_number,chunk_index,section_heading,content")
+ .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata")
.eq("document_id", documentId)
.order("chunk_index", { ascending: true })
- .limit(24),
- supabase
+ .range(start, start + 999);
+ if (error) throw new Error(error.message);
+ chunks.push(...(data ?? []));
+ if (!data || data.length < 1000) break;
+ }
+
+ for (let start = 0; ; start += 1000) {
+ const { data, error } = await supabase
.from("document_images")
- .select("id,page_number,caption,image_type,labels")
+ .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata")
.eq("document_id", documentId)
.eq("searchable", true)
.order("clinical_relevance_score", { ascending: false })
- .limit(12),
- ]);
+ .range(start, start + 999);
+ if (error) throw new Error(error.message);
+ images.push(...(data ?? []));
+ if (!data || data.length < 1000) break;
+ }
- if (chunksResult.error) throw new Error(chunksResult.error.message);
- if (imagesResult.error) throw new Error(imagesResult.error.message);
- return {
- chunks: chunksResult.data ?? [],
- images: imagesResult.data ?? [],
- };
+ return { chunks, images };
}
async function processJob(job: JobRow) {
@@ -450,7 +628,7 @@ async function processJob(job: JobRow) {
);
const metrics = extractionMetrics(extracted, skippedImages, imageSkipReasons, imageTypeCounts);
- await updateJob(job.id, { stage: "summarising and labelling", progress: 96 });
+ await updateJob(job.id, { stage: "summarising and labelling", progress: 95 });
const enrichmentRows = await loadEnrichmentRows(job.document_id);
await upsertDocumentEnrichment({
supabase,
@@ -458,7 +636,15 @@ async function processJob(job: JobRow) {
chunks: enrichmentRows.chunks,
images: enrichmentRows.images,
});
+ await updateJob(job.id, { stage: "building structured memory", progress: 98 });
+ const deepMemory = await upsertDocumentDeepMemory({
+ supabase,
+ document: job.documents,
+ chunks: enrichmentRows.chunks,
+ images: enrichmentRows.images,
+ });
+ const indexedAt = new Date().toISOString();
await updateDocument(job.document_id, {
status: "indexed",
page_count: extracted.pages.length,
@@ -467,7 +653,14 @@ async function processJob(job: JobRow) {
error_message: null,
metadata: {
...(job.documents.metadata ?? {}),
- indexed_at: new Date().toISOString(),
+ indexed_at: indexedAt,
+ rag_enrichment_version: ragEnrichmentVersion,
+ rag_indexing_version: ragDeepMemoryVersion,
+ rag_memory_version: ragDeepMemoryVersion,
+ rag_memory_updated_at: indexedAt,
+ rag_enrichment_updated_at: indexedAt,
+ section_count: deepMemory.sections.length,
+ memory_card_count: deepMemory.memoryCards.length,
extraction_quality: extracted.pages.length > 0 && metrics.text_character_count >= 80 ? "good" : "partial",
embedding_model: env.OPENAI_EMBEDDING_MODEL,
...metrics,
diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py
index a561108d7..421fab7b7 100644
--- a/worker/python/extract_pdf_assets.py
+++ b/worker/python/extract_pdf_assets.py
@@ -1,6 +1,7 @@
import io
import json
import os
+import re
import sys
from statistics import median
@@ -71,6 +72,24 @@ def save_page_crop(page, rect, output_dir, file_name, source_kind, metadata):
}
+def expanded_rect(rect, page_rect, x_padding=4, y_padding=4):
+ return fitz.Rect(
+ max(rect.x0 - x_padding, page_rect.x0),
+ max(rect.y0 - y_padding, page_rect.y0),
+ min(rect.x1 + x_padding, page_rect.x1),
+ min(rect.y1 + y_padding, page_rect.y1),
+ )
+
+
+def rect_intersection_ratio(a, b):
+ intersection = a & b
+ if intersection.is_empty:
+ return 0
+ intersection_area = intersection.width * intersection.height
+ smaller_area = max(min(a.width * a.height, b.width * b.height), 1)
+ return intersection_area / smaller_area
+
+
def merge_rects(rects, page_rect):
if not rects:
return None
@@ -81,13 +100,441 @@ def merge_rects(rects, page_rect):
return fitz.Rect(x0, y0, x1, y1)
-def likely_table_rects(page):
+def clean_cell(value):
+ return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def table_rows_to_markdown(rows):
+ cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)]
+ if not cleaned:
+ return ""
+
+ column_count = max(len(row) for row in cleaned)
+ padded = [row + [""] * (column_count - len(row)) for row in cleaned]
+ header = padded[0]
+ separator = ["---"] * column_count
+ body = padded[1:]
+
+ def markdown_row(row):
+ escaped = [cell.replace("|", "\\|") for cell in row]
+ return "| " + " | ".join(escaped) + " |"
+
+ return "\n".join([markdown_row(header), markdown_row(separator), *[markdown_row(row) for row in body]])
+
+
+def table_columns_from_rows(rows):
+ cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)]
+ if not cleaned:
+ return []
+ return cleaned[0]
+
+
+def text_grid_rows(text):
+ rows = []
+ for line in (text or "").splitlines():
+ cleaned = re.sub(r"\s+", " ", line).strip()
+ if not cleaned:
+ continue
+ cells = [clean_cell(cell) for cell in re.split(r"\s{2,}|\t|\|", line) if clean_cell(cell)]
+ rows.append(cells if len(cells) > 1 else [cleaned])
+ return rows
+
+
+def extract_table_payload(table):
+ try:
+ rows = table.extract()
+ cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)]
+ column_count = max((len(row) for row in cleaned), default=0)
+ return {
+ "text": table_rows_to_markdown(rows),
+ "rows": cleaned[:80],
+ "columns": table_columns_from_rows(rows)[:24],
+ "accessible_markdown": table_rows_to_markdown(rows),
+ "row_count": len(cleaned),
+ "column_count": column_count,
+ }
+ except Exception:
+ return {"text": "", "rows": [], "columns": [], "accessible_markdown": "", "row_count": 0, "column_count": 0}
+
+
+def extract_table_text(table):
+ return extract_table_payload(table)["text"]
+
+
+def split_table_heading(text):
+ cleaned = re.sub(r"\s+", " ", text or "").strip(" :-\u2013\u2014")
+ if not cleaned:
+ return None, None
+
+ patterns = (
+ r"(?i)\b(table\s+\d+[a-z]?)\s*[:.\-\u2013\u2014]?\s*(.+)",
+ r"(?i)\b(appendix\s+\d+[a-z]?)\s*[:.\-\u2013\u2014]?\s*(.+)",
+ )
+ for pattern in patterns:
+ match = re.search(pattern, cleaned)
+ if match:
+ label = re.sub(r"\s+", " ", match.group(1)).title()
+ title = re.sub(r"\s+", " ", match.group(2)).strip(" :-\u2013\u2014")
+ return label, title[:240] if title else None
+
+ section_match = re.search(r"(?i)(?:^|\s)\d+\.?\s+(roles and responsibilities)\b", cleaned)
+ if section_match:
+ return None, section_match.group(1).title()
+
+ if re.fullmatch(r"(?i)roles and responsibilities", cleaned):
+ return None, "Roles and responsibilities"
+
+ known_section_headings = (
+ "recommended pharmacological treatment options",
+ "*maximum dose refers to total maximum oral and im dose (including regular)",
+ )
+ if cleaned.lower() in known_section_headings:
+ return None, cleaned[:180]
+
+ return None, None
+
+
+def nearby_table_heading(page, rect, table_text):
+ blocks = sorted(page.get_text("blocks") or [], key=lambda block: (block[1], block[0]))
+ candidates = []
+ for block in blocks:
+ if len(block) < 5:
+ continue
+ x0, y0, x1, y1, text = block[:5]
+ if y1 > rect.y0 + 16 or y1 < rect.y0 - 140:
+ continue
+ if x1 < rect.x0 - 40 or x0 > rect.x1 + 40:
+ continue
+ cleaned = re.sub(r"\s+", " ", str(text)).strip()
+ if cleaned:
+ candidates.append(cleaned)
+
+ for candidate in reversed(candidates):
+ label, title = split_table_heading(candidate)
+ if label or title:
+ return label, title, candidate
+
+ lowered_table = table_text.lower()
+ if "role" in lowered_table and "responsibility" in lowered_table:
+ return None, "Roles and responsibilities", "Roles and responsibilities"
+ return None, None, ""
+
+
+def clinical_table_score(text):
+ lowered = text.lower()
+ score = 0
+ keywords = (
+ "management",
+ "monitor",
+ "observation",
+ "medication",
+ "dose",
+ "benzodiazepine",
+ "antipsychotic",
+ "intramuscular",
+ "oral",
+ "risk",
+ "escalat",
+ "responsibil",
+ "patient",
+ "score",
+ "frequency",
+ "post im",
+ "post po",
+ "appendix",
+ "table",
+ )
+ for keyword in keywords:
+ if keyword in lowered:
+ score += 1
+ return score
+
+
+def table_text_metrics(table_text):
+ text = table_text or ""
+ lines = [line for line in text.splitlines() if line.strip()]
+ return {
+ "bars": text.count("|"),
+ "line_count": len(lines),
+ "clinical_score": clinical_table_score(text),
+ }
+
+
+def table_role_for_candidate(label, title, table_text):
+ combined = " ".join([label or "", title or "", table_text or ""]).lower()
+ clinical_markers = (
+ "management",
+ "monitor",
+ "observation",
+ "medication",
+ "dose",
+ "benzodiazepine",
+ "antipsychotic",
+ "intramuscular",
+ "oral",
+ "risk",
+ "escalat",
+ "patient",
+ "score",
+ "frequency",
+ "post im",
+ "post po",
+ "treatment",
+ "assessment",
+ "side effect",
+ "contraindicat",
+ "responsibil",
+ )
+ admin_markers = (
+ "authorisation date",
+ "published date",
+ "version control",
+ "document owner",
+ "approval",
+ "endorsed",
+ "review date",
+ "contact person",
+ "policy sponsor",
+ "compliance monitoring",
+ "amendment",
+ "amended",
+ "revision",
+ "reviewed",
+ "superseded",
+ "section 4.0",
+ "section 13",
+ )
+ reference_markers = (
+ "references",
+ "relevant standards",
+ "documents support",
+ "bibliography",
+ "legislation",
+ "associated documents",
+ )
+
+ if title and title.lower() == "roles and responsibilities":
+ return "clinical"
+ if any(marker in combined for marker in reference_markers):
+ return "reference"
+ if any(marker in combined for marker in admin_markers):
+ return "admin"
+ if any(marker in combined for marker in clinical_markers):
+ return "clinical"
+ if label and label.lower().startswith(("table", "appendix")):
+ return "unclear"
+ return "unclear"
+
+
+def table_candidate_confidence(label, title, table_text, row_count=0, column_count=0):
+ metrics = table_text_metrics(table_text)
+ confidence = 0.2
+ if label:
+ confidence += 0.18
+ if title:
+ confidence += 0.14
+ if row_count >= 2:
+ confidence += 0.16
+ if row_count >= 4:
+ confidence += 0.1
+ if column_count >= 2:
+ confidence += 0.16
+ if column_count >= 3:
+ confidence += 0.08
+ if metrics["bars"] >= 8:
+ confidence += 0.12
+ if metrics["line_count"] >= 3:
+ confidence += 0.08
+ if metrics["clinical_score"] >= 3:
+ confidence += 0.08
+ return round(min(confidence, 0.99), 2)
+
+
+def real_table_candidate(label, title, table_text, row_count=0, column_count=0):
+ metrics = table_text_metrics(table_text)
+ combined = " ".join([label or "", title or "", table_text or ""]).lower()
+ known_table_heading = bool(label) or (title or "").lower() in (
+ "roles and responsibilities",
+ "recommended pharmacological treatment options",
+ )
+
+ if known_table_heading and metrics["line_count"] >= 2:
+ return True
+ if row_count >= 2 and column_count >= 2 and metrics["line_count"] >= 2:
+ return True
+ if metrics["bars"] >= 6 and metrics["line_count"] >= 3:
+ return True
+ if "role" in combined and "responsibility" in combined and metrics["line_count"] >= 3:
+ return True
+ if any(marker in combined for marker in ("version control", "authorisation date", "published date")):
+ return metrics["line_count"] >= 3
+ return False
+
+
+def likely_table_candidates(page):
+ candidates = []
try:
tables = page.find_tables()
- return [table.bbox for table in tables.tables]
+ for index, table in enumerate(tables.tables):
+ rect = fitz.Rect(table.bbox)
+ payload = extract_table_payload(table)
+ table_text = payload["text"]
+ label, title, heading_text = nearby_table_heading(page, rect, table_text)
+ row_count = payload["row_count"]
+ column_count = payload["column_count"]
+ if not real_table_candidate(label, title, table_text, row_count, column_count):
+ continue
+ role = table_role_for_candidate(label, title, table_text)
+ candidates.append(
+ {
+ "rect": rect,
+ "table_text": table_text,
+ "table_label": label,
+ "table_title": title,
+ "table_role": role,
+ "table_confidence": table_candidate_confidence(label, title, table_text, row_count, column_count),
+ "table_rows": payload.get("rows", []),
+ "table_columns": payload.get("columns", []),
+ "accessible_table_markdown": payload.get("accessible_markdown") or table_text,
+ "row_count": row_count,
+ "column_count": column_count,
+ "heading_text": heading_text,
+ "extraction_method": "pymupdf_find_tables",
+ "table_index": index + 1,
+ }
+ )
except Exception:
+ pass
+
+ return candidates
+
+
+def fallback_table_candidates(page, existing_rects):
+ blocks = sorted(page.get_text("blocks") or [], key=lambda block: (block[1], block[0]))
+ text_blocks = []
+ for block in blocks:
+ if len(block) < 5:
+ continue
+ rect = fitz.Rect(block[:4])
+ raw_text = str(block[4])
+ text = re.sub(r"\s+", " ", raw_text).strip()
+ if not text or rect.height < 10:
+ continue
+ text_blocks.append((rect, text, raw_text))
+
+ candidates = []
+ active = []
+ for rect, text, raw_text in text_blocks:
+ score = clinical_table_score(text)
+ has_columns = raw_text.count(" ") >= 2 or "\t" in raw_text or "|" in raw_text
+ table_role = table_role_for_candidate(None, None, text)
+ if score >= 2 or has_columns or table_role in ("admin", "reference"):
+ active.append((rect, text))
+ continue
+
+ if active:
+ candidates.extend(active)
+ active = []
+ if active:
+ candidates.extend(active)
+
+ if not candidates:
return []
+ rects = [rect for rect, _ in candidates]
+ region = merge_rects(rects, page.rect)
+ text = "\n".join(text for _, text in candidates)
+ if not region:
+ return []
+ if any(rect_intersection_ratio(region, existing) > 0.45 for existing in existing_rects):
+ return []
+
+ label, title, heading_text = nearby_table_heading(page, region, text)
+ inferred_columns = max((len(re.split(r"\s{2,}|\t|\|", line.strip())) for line in text.splitlines() if line.strip()), default=0)
+ inferred_rows = len([line for line in text.splitlines() if line.strip()])
+ rows = text_grid_rows(text)
+ role = table_role_for_candidate(label, title, text)
+ if not real_table_candidate(label, title, text, inferred_rows, inferred_columns):
+ return []
+ return [
+ {
+ "rect": region,
+ "table_text": text[:8000],
+ "table_label": label,
+ "table_title": title,
+ "table_role": role,
+ "table_confidence": table_candidate_confidence(label, title, text, inferred_rows, inferred_columns),
+ "table_rows": rows[:80],
+ "table_columns": (rows[0] if rows else [])[:24],
+ "accessible_table_markdown": table_rows_to_markdown(rows) if rows and max((len(row) for row in rows), default=0) > 1 else text[:8000],
+ "row_count": inferred_rows,
+ "column_count": inferred_columns,
+ "heading_text": heading_text,
+ "extraction_method": "text_grid_heuristic",
+ "table_index": len(existing_rects) + 1,
+ }
+ ]
+
+
+def merge_related_table_candidates(candidates, page_rect):
+ if len(candidates) < 2:
+ return candidates
+
+ merged = []
+ used = set()
+ for index, candidate in enumerate(candidates):
+ if index in used:
+ continue
+
+ label = (candidate.get("table_label") or "").lower()
+ if not label.startswith("appendix"):
+ merged.append(candidate)
+ continue
+
+ group = [candidate]
+ for other_index, other in enumerate(candidates):
+ if other_index == index or other_index in used:
+ continue
+ title = (other.get("table_title") or "").lower()
+ text = (other.get("table_text") or "").lower()
+ vertical_gap = other["rect"].y0 - candidate["rect"].y1
+ if other["rect"].y0 >= candidate["rect"].y0 and vertical_gap < 160 and (
+ "recommended pharmacological treatment options" in title
+ or "maximum dose" in title
+ or "intramuscular" in text
+ or "benzodiazepine" in text
+ ):
+ group.append(other)
+ used.add(other_index)
+
+ if len(group) > 1:
+ rect = merge_rects([item["rect"] for item in group], page_rect)
+ merged_text = "\n\n".join(item.get("table_text") or "" for item in group).strip()
+ merged_rows = []
+ for item in group:
+ merged_rows.extend(item.get("table_rows") or [])
+ candidate = {
+ **candidate,
+ "rect": rect,
+ "table_text": merged_text,
+ "table_role": table_role_for_candidate(
+ candidate.get("table_label"),
+ candidate.get("table_title"),
+ merged_text,
+ ),
+ "table_confidence": max(item.get("table_confidence") or 0 for item in group),
+ "table_rows": merged_rows[:120],
+ "table_columns": (merged_rows[0] if merged_rows else candidate.get("table_columns") or [])[:24],
+ "accessible_table_markdown": table_rows_to_markdown(merged_rows) if merged_rows else merged_text,
+ "row_count": sum(item.get("row_count") or 0 for item in group),
+ "column_count": max(item.get("column_count") or 0 for item in group),
+ "extraction_method": "merged_appendix_tables",
+ }
+ merged.append(candidate)
+ used.add(index)
+
+ return merged
+
def likely_vector_region(page):
drawings = page.get_drawings()
@@ -128,6 +575,35 @@ def fallback_visual_region(page):
if page_visual_weight(page) < 8:
return None
+ text = (page.get_text("text", sort=True) or "").lower()
+ admin_markers = (
+ "authorisation",
+ "published date",
+ "document owner",
+ "version control",
+ "relevant standards",
+ "references",
+ "compliance monitoring",
+ )
+ high_value_markers = (
+ "dose",
+ "intramuscular",
+ "benzodiazepine",
+ "antipsychotic",
+ "lorazepam",
+ "olanzapine",
+ "score",
+ "observation",
+ "post im",
+ "post po",
+ )
+ if any(marker in text for marker in ("relevant standards", "references", "document owner", "compliance monitoring")):
+ return None
+ if any(marker in text for marker in admin_markers) and not any(marker in text for marker in high_value_markers):
+ return None
+ if ("version control" in text or "authorisation date" in text) and "recommended pharmacological treatment options" not in text:
+ return None
+
top_margin = page.rect.height * 0.08
bottom_margin = page.rect.height * 0.08
return fitz.Rect(
@@ -199,25 +675,45 @@ def extract(pdf_path, output_dir):
page_image_count += 1
crop_index = 1
- for table_rect in likely_table_rects(page):
+ table_rects = []
+ table_candidates = likely_table_candidates(page)
+ table_candidates.extend(fallback_table_candidates(page, [candidate["rect"] for candidate in table_candidates]))
+ table_candidates = merge_related_table_candidates(table_candidates, page.rect)
+ for table_candidate in table_candidates:
+ table_rect = expanded_rect(table_candidate["rect"], page.rect, 4, 4)
crop = save_page_crop(
page,
- fitz.Rect(table_rect),
+ table_rect,
output_dir,
f"page-{page_number}-table-crop-{crop_index}.png",
- "diagram_crop",
+ "table_crop",
{
"pageNumber": page_number,
- "source_kind": "diagram_crop",
+ "source_kind": "table_crop",
"candidate_type": "table",
+ "table_label": table_candidate.get("table_label"),
+ "table_title": table_candidate.get("table_title"),
+ "table_text": (table_candidate.get("table_text") or "")[:8000],
+ "table_role": table_candidate.get("table_role"),
+ "table_confidence": table_candidate.get("table_confidence"),
+ "table_rows": table_candidate.get("table_rows") or [],
+ "table_columns": table_candidate.get("table_columns") or [],
+ "accessible_table_markdown": (table_candidate.get("accessible_table_markdown") or table_candidate.get("table_text") or "")[:8000],
+ "row_count": table_candidate.get("row_count"),
+ "column_count": table_candidate.get("column_count"),
+ "heading_text": table_candidate.get("heading_text"),
+ "bbox": rect_payload(table_rect),
+ "extraction_method": table_candidate.get("extraction_method"),
+ "table_index": table_candidate.get("table_index"),
},
)
if crop:
images.append(crop)
+ table_rects.append(table_rect)
crop_index += 1
vector_region = likely_vector_region(page)
- if vector_region:
+ if vector_region and not any(rect_intersection_ratio(vector_region, table_rect) > 0.45 for table_rect in table_rects):
crop = save_page_crop(
page,
vector_region,
@@ -236,7 +732,9 @@ def extract(pdf_path, output_dir):
if page_image_count == 0 and crop_index == 1:
fallback_region = fallback_visual_region(page)
- if fallback_region:
+ if fallback_region and not any(
+ rect_intersection_ratio(fallback_region, table_rect) > 0.45 for table_rect in table_rects
+ ):
crop = save_page_crop(
page,
fallback_region,
@@ -256,8 +754,13 @@ def extract(pdf_path, output_dir):
if __name__ == "__main__":
- if len(sys.argv) != 3:
- print("Usage: extract_pdf_assets.py input.pdf output_dir", file=sys.stderr)
+ if len(sys.argv) not in (3, 4):
+ print("Usage: extract_pdf_assets.py input.pdf output_dir [output.json]", file=sys.stderr)
sys.exit(1)
- print(json.dumps(extract(sys.argv[1], sys.argv[2])))
+ result = extract(sys.argv[1], sys.argv[2])
+ if len(sys.argv) == 4:
+ with open(sys.argv[3], "w", encoding="utf-8") as handle:
+ json.dump(result, handle)
+ else:
+ print(json.dumps(result))