From bc7d7e66f57b6a13761c126106d2741e586958b3 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 28 May 2026 13:00:52 +0800 Subject: [PATCH 1/7] Document upload shortcut workflow --- AGENTS.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 1bd3f6f39..5e25aa933 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,3 +27,103 @@ This version has breaking changes — APIs, conventions, and file structure may - Older unused project ref `qjgitjyhxrwxsrydablr` belongs to `Database`; treat it as stale and do not use it. - Run `npm run check:supabase-project` after changing Supabase env values. + + + +# `upload` shortcut + +When the user types exactly: + +upload + +as the entire task message, treat it as a shortcut for the safe Git handoff workflow below. + +The goal is to leave useful completed work safely committed and, where safe, pushed to the current feature branch. The goal is not to merge into main, delete branches, discard work, force-push, close PRs, deploy, or perform destructive cleanup without explicit user confirmation. + +## Protected and base branches + +Treat `main`, `master`, `develop`, and `release/*` as protected/base branches for this workflow. + +If `upload` is run while on `main`, create or use a branch named exactly `temporary` before staging, committing, or pushing: + +- If neither local `temporary` nor `origin/temporary` exists, run `git switch -c temporary`. +- If local `temporary` exists and is not checked out in another worktree, switch to it only when it is clearly safe. +- If `origin/temporary` exists, use it only when it is clearly the matching intended branch. +- If any `temporary` branch state is ambiguous, diverged, checked out elsewhere, or unsafe, stop and ask instead of overwriting. + +If already on a non-protected feature branch, continue using that branch. + +## Required inspection + +Start with read-only inspection before making changes. Check: + +- Current branch or detached HEAD state +- `git status` +- Staged, unstaged, and untracked files +- Recent commits relevant to the current branch +- Remote configuration and upstream branch +- Whether the branch is ahead, behind, or diverged +- Whether the current branch appears protected/base +- Other Git worktrees, if detectable +- Available checks such as tests, lint, type check, or build scripts +- Existing branch, commit, PR, and release-flow conventions + +Do not assume branch names, remotes, package managers, test commands, deployment targets, or project structure. Inspect first. + +## Safe actions allowed without further confirmation + +When the repository state makes it clearly safe, you may: + +- Stage coherent completed changes that clearly belong together +- Create one or more logical commits with clear messages based on the diff +- Fast-forward pull only when there are no local commits or conflict risks +- Push the current non-protected feature branch if it has a valid upstream +- Set an upstream for the current feature branch only when the correct remote and branch name are obvious +- Leave the worktree clean by committing safe completed changes + +## Actions requiring explicit confirmation + +Do not perform these without asking the user first: + +- `git reset --hard` +- `git clean -fd` or other destructive cleanup +- Discarding, overwriting, or reverting uncommitted changes +- Deleting local or remote branches +- Renaming branches +- Force-pushing +- Rebasing a shared/public branch +- Resolving divergent branch history +- Merging into `main`, `master`, `develop`, `release/*`, or any protected/base branch +- Closing pull requests +- Changing GitHub default branch, branch protection, repository settings, or deployment settings +- Modifying production data or deployment configuration +- Committing secrets, credentials, tokens, private keys, or sensitive local configuration +- Updating branch references where the correct replacement branch is ambiguous + +If any of these seem necessary, stop and report what is risky, why it is risky, the recommended next step, and the exact confirmation needed. + +## Mixed, suspicious, or unsafe changes + +Do not automatically commit files that look like `.env` files, credentials, secrets, logs, caches, build artifacts, editor or OS files, temporary/debug files, or generated files not normally committed by this project. Report only the path and concern for possible secrets; never print secret values. + +If changes appear unrelated, incomplete, experimental, or WIP, do not commit everything together automatically. Commit only clearly coherent completed changes when safe; otherwise summarize the groups and ask what should be included. + +## Branch cleanup and reference updates + +If stale, inappropriate, merged, or unnecessary branches are detected, list cleanup candidates but do not delete or rename branches automatically. + +Before recommending deletion or rename, audit accessible references including `.github/workflows/*`, CI/CD config, deployment config, scripts, package scripts, docs, release notes or release scripts, safe environment/config files, branch-specific config, open PR metadata if accessible, and GitHub branch protection/default branch metadata if safely accessible. + +Update repo-tracked references to a renamed or replacement branch only when the old branch reference is clearly found, the replacement is obvious, the change is low-risk, and the user has approved the branch rename or deletion. If the replacement is unclear, report the reference and ask what it should point to. + +## Syncing and verification + +Do not rebase, merge, or resolve remote divergence automatically. Fast-forward pulls are allowed only when clearly safe. Push only the current non-protected feature branch when clearly safe. + +Run the smallest relevant checks that are available and appropriate, such as tests, lint, type check, or build checks. Do not claim checks passed unless they were actually run. If checks cannot be run, explain why and state the command that would normally be used. + +## Final report + +After completing `upload`, summarize the current branch and worktree state, whether the worktree is clean, what changed, files committed, commit hash and message if created, whether anything was pushed, remote branch and likely PR target, checks run and results, checks not run and why, branch cleanup candidates, branch references found or updated, risky actions skipped, and exact confirmation needed for any recommended follow-up. + + From f134a73b50699a11728ea23414e9cf7ba20c67e8 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 28 May 2026 13:08:08 +0800 Subject: [PATCH 2/7] Clarify upload branch fallback --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 5e25aa933..bd2f576bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ The goal is to leave useful completed work safely committed and, where safe, pus Treat `main`, `master`, `develop`, and `release/*` as protected/base branches for this workflow. -If `upload` is run while on `main`, create or use a branch named exactly `temporary` before staging, committing, or pushing: +If `upload` is run while on `main`, automatically create or use a branch named exactly `temporary` before staging, committing, or pushing, then continue the upload workflow from `temporary`: - If neither local `temporary` nor `origin/temporary` exists, run `git switch -c temporary`. - If local `temporary` exists and is not checked out in another worktree, switch to it only when it is clearly safe. From 2d33a783c219d19e144a0286cfcdaa9b603d7846 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 28 May 2026 13:10:13 +0800 Subject: [PATCH 3/7] Handle Supabase project checks in UI smoke tests --- src/app/api/documents/[id]/route.ts | 178 ++++++++++++- src/components/ClinicalDashboard.tsx | 78 ++++++ src/components/DocumentManagementActions.tsx | 251 +++++++++++++++++++ src/components/DocumentViewer.tsx | 39 ++- src/lib/rag.ts | 16 ++ tests/private-access-routes.test.ts | 164 +++++++++++- 6 files changed, 713 insertions(+), 13 deletions(-) create mode 100644 src/components/DocumentManagementActions.tsx diff --git a/src/app/api/documents/[id]/route.ts b/src/app/api/documents/[id]/route.ts index 05be91637..e59dab3b5 100644 --- a/src/app/api/documents/[id]/route.ts +++ b/src/app/api/documents/[id]/route.ts @@ -1,12 +1,58 @@ import { NextResponse } from "next/server"; +import { z } from "zod"; import { getDemoDocumentPayload } from "@/lib/demo-data"; -import { isDemoMode } from "@/lib/env"; -import { jsonError } from "@/lib/http"; +import { env, isDemoMode } from "@/lib/env"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { invalidateRagCachesForOwner } from "@/lib/rag"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; +const renameSchema = z.object({ + title: z.string().trim().min(1).max(180), +}); + +function safeMetadata(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function storageWarningsFrom(error: unknown, label: string) { + const message = error && typeof error === "object" && "message" in error ? String(error.message) : "Storage cleanup failed."; + return `${label}: ${message}`; +} + +async function removeStorageObjects(args: { + supabase: ReturnType; + sourcePath: string | null; + imagePaths: string[]; +}) { + const warnings: string[] = []; + let storageRemoved = 0; + + if (args.sourcePath) { + const sourceRemove = await args.supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([args.sourcePath]); + if (sourceRemove.error) { + warnings.push(storageWarningsFrom(sourceRemove.error, "Source PDF")); + } else { + storageRemoved += sourceRemove.data?.length ?? 0; + } + } + + const uniqueImagePaths = Array.from(new Set(args.imagePaths.filter(Boolean))); + for (let start = 0; start < uniqueImagePaths.length; start += 1000) { + const paths = uniqueImagePaths.slice(start, start + 1000); + const imageRemove = await args.supabase.storage.from(env.SUPABASE_IMAGE_BUCKET).remove(paths); + if (imageRemove.error) { + warnings.push(storageWarningsFrom(imageRemove.error, "Extracted images")); + } else { + storageRemoved += imageRemove.data?.length ?? 0; + } + } + + return { storageRemoved, storageWarnings: warnings }; +} + export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { try { const { id } = await params; @@ -89,3 +135,131 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: return jsonError(error); } } + +export async function PATCH(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + if (isDemoMode()) { + return NextResponse.json({ error: "Demo documents cannot be renamed." }, { status: 400 }); + } + + const parsed = renameSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) { + throw new PublicApiError("Enter a document title between 1 and 180 characters."); + } + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const { data: document, error: documentError } = await supabase + .from("documents") + .select("id,owner_id,title,file_name,storage_path,content_hash,metadata") + .eq("id", id) + .eq("owner_id", user.id) + .maybeSingle(); + + if (documentError) throw new Error(documentError.message); + if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 }); + if (document.title.trim() === parsed.data.title) { + throw new PublicApiError("Document title is unchanged."); + } + + const metadata = safeMetadata(document.metadata); + const { data: updated, error: updateError } = await supabase + .from("documents") + .update({ + title: parsed.data.title, + metadata: { + ...metadata, + renamed_at: new Date().toISOString(), + previous_title: document.title, + original_file_name: metadata.original_file_name ?? document.file_name, + }, + }) + .eq("id", id) + .eq("owner_id", user.id) + .select("*") + .single(); + + if (updateError) throw new Error(updateError.message); + invalidateRagCachesForOwner(user.id); + return NextResponse.json({ document: updated }); + } catch (error) { + if (error instanceof AuthenticationError) { + return unauthorizedResponse(); + } + return jsonError(error); + } +} + +export async function DELETE(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + if (isDemoMode()) { + return NextResponse.json({ error: "Demo documents cannot be deleted." }, { status: 400 }); + } + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const { data: document, error: documentError } = await supabase + .from("documents") + .select("id,owner_id,title,storage_path") + .eq("id", id) + .eq("owner_id", user.id) + .maybeSingle(); + + if (documentError) throw new Error(documentError.message); + if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 }); + + const { data: activeJobs, error: activeJobsError } = await supabase + .from("ingestion_jobs") + .select("id,status") + .eq("document_id", id) + .eq("status", "processing") + .limit(1); + + if (activeJobsError) throw new Error(activeJobsError.message); + if ((activeJobs ?? []).length > 0) { + throw new PublicApiError("Document is currently indexing. Stop or wait for the worker before deleting.", 409); + } + + const [{ data: images, error: imagesError }, { data: chunks, error: chunksError }] = await Promise.all([ + supabase.from("document_images").select("storage_path").eq("document_id", id), + supabase.from("document_chunks").select("id").eq("document_id", id), + ]); + + if (imagesError) throw new Error(imagesError.message); + if (chunksError) throw new Error(chunksError.message); + + const chunkIds = (chunks ?? []).map((chunk) => chunk.id).filter(Boolean); + if (chunkIds.length > 0) { + const { error: queryDeleteError } = await supabase + .from("rag_queries") + .delete() + .eq("owner_id", user.id) + .overlaps("source_chunk_ids", chunkIds); + if (queryDeleteError) throw new Error(queryDeleteError.message); + } + + const { error: deleteError } = await supabase.from("documents").delete().eq("id", id).eq("owner_id", user.id); + if (deleteError) throw new Error(deleteError.message); + + const cleanup = await removeStorageObjects({ + supabase, + sourcePath: document.storage_path, + imagePaths: (images ?? []).map((image) => image.storage_path).filter(Boolean), + }); + + invalidateRagCachesForOwner(user.id); + return NextResponse.json({ + deleted: true, + documentId: id, + storageRemoved: cleanup.storageRemoved, + storageWarnings: cleanup.storageWarnings, + }); + } catch (error) { + if (error instanceof AuthenticationError) { + return unauthorizedResponse(); + } + return jsonError(error); + } +} diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index a3a6d3204..b959b1486 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -32,6 +32,7 @@ import { X, } from "lucide-react"; import { FormEvent, ReactNode, useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react"; +import { DocumentManagementActions, type DocumentDeleteResult } from "@/components/DocumentManagementActions"; import { documentCitationHref, formatCompactCitationLabel, formatCitationLabel } from "@/lib/citations"; import { extractSafetyFindings, formatSafetyFindingLabel } from "@/lib/clinical-safety"; import { clearCachedSignedUrl, getCachedSignedUrl, setCachedSignedUrl } from "@/lib/signed-url-cache"; @@ -1442,10 +1443,16 @@ function DocumentDrawer({ documents, selectedDocumentIds, onToggleScope, + onDocumentRenamed, + onDocumentDeleted, + canManageDocuments, }: { documents: ClinicalDocument[]; selectedDocumentIds: string[]; onToggleScope: (documentId: string) => void; + onDocumentRenamed: (document: ClinicalDocument) => void; + onDocumentDeleted: (result: DocumentDeleteResult) => void; + canManageDocuments: boolean; }) { const [filter, setFilter] = useState(""); const filtered = documents.filter((document) => { @@ -1515,6 +1522,12 @@ function DocumentDrawer({
+ + +
+ + {mode && ( +
+
+
+
+

+ {mode === "rename" ? "Rename document" : "Permanently delete document"} +

+

+ {mode === "rename" + ? "The original file name and storage path will stay unchanged." + : "This removes the source, extracted evidence, images, labels, summaries, and related query logs."} +

+
+ +
+ + {error && ( +
{error}
+ )} + + {mode === "rename" ? ( +
+ +

Original file: {document.file_name}

+
+ + +
+
+ ) : ( +
+
+
+ +

This action cannot be undone.

+
+
+ +

{document.title}

+
+ + +
+
+ )} +
+
+ )} + + ); +} diff --git a/src/components/DocumentViewer.tsx b/src/components/DocumentViewer.tsx index d86fc05fc..d3fc57c3c 100644 --- a/src/components/DocumentViewer.tsx +++ b/src/components/DocumentViewer.tsx @@ -3,6 +3,7 @@ /* eslint-disable @next/next/no-img-element */ import Link from "next/link"; +import { useRouter } from "next/navigation"; import { AlertCircle, ArrowLeft, @@ -46,6 +47,7 @@ import { clearCachedSignedUrl, getCachedSignedUrl, setCachedSignedUrl } from "@/ import { formatClinicalDate, normalizeSourceMetadata, sourceStatusLabel } from "@/lib/source-metadata"; import { useAuthSession } from "@/lib/supabase/client"; import { SafeBoldText } from "@/components/SafeBoldText"; +import { DocumentManagementActions } from "@/components/DocumentManagementActions"; import type { ClinicalDocument, RagAnswer } from "@/lib/types"; type PageRow = { @@ -564,6 +566,7 @@ export function DocumentViewer({ initialPage: number; chunkId?: string; }) { + const router = useRouter(); const [document, setDocument] = useState(null); const [pages, setPages] = useState([]); const [images, setImages] = useState([]); @@ -789,6 +792,12 @@ export function DocumentViewer({ setLoadingDocument(true); setPreviewAttempt((current) => current + 1); }; + const handleDocumentRenamed = (updatedDocument: ClinicalDocument) => { + setDocument((current) => (current?.id === updatedDocument.id ? { ...current, ...updatedDocument } : current)); + }; + const handleDocumentDeleted = () => { + router.push("/"); + }; return (
@@ -830,16 +839,26 @@ export function DocumentViewer({

- +
+ {readyDocument && ( + + )} + +
diff --git a/src/lib/rag.ts b/src/lib/rag.ts index b0ebea843..49c36ed51 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -436,6 +436,22 @@ function setCachedSearch(args: SearchChunksArgs, results: SearchResult[], teleme } } +export function invalidateRagCachesForOwner(ownerId?: string | null) { + if (!ownerId) { + answerCache.clear(); + searchCache.clear(); + return; + } + + const prefix = `${ownerId}|`; + for (const key of answerCache.keys()) { + if (key.startsWith(prefix)) answerCache.delete(key); + } + for (const key of searchCache.keys()) { + if (key.startsWith(prefix)) searchCache.delete(key); + } +} + async function insertRagQuery(row: Record) { const supabase = createAdminClient(); await supabase.from("rag_queries").insert(row); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index ff0b81115..448721c04 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -13,11 +13,13 @@ type QueryFilter = { column: string; value: unknown }; type QueryInFilter = { column: string; values: unknown[] }; type QueryCall = { table: string; - operation: "select" | "insert"; + operation: "select" | "insert" | "update" | "delete"; selected?: string; filters: QueryFilter[]; inFilters: QueryInFilter[]; + overlapsFilters: QueryInFilter[]; insertPayload?: unknown; + updatePayload?: unknown; limitCount?: number; maybeSingle: boolean; single: boolean; @@ -49,6 +51,17 @@ class QueryBuilder implements PromiseLike { return this; } + update(payload: unknown) { + this.call.operation = "update"; + this.call.updatePayload = payload; + return this; + } + + delete() { + this.call.operation = "delete"; + return this; + } + eq(column: string, value: unknown) { this.call.filters.push({ column, value }); return this; @@ -59,6 +72,11 @@ class QueryBuilder implements PromiseLike { return this; } + overlaps(column: string, values: unknown[]) { + this.call.overlapsFilters.push({ column, values }); + return this; + } + order() { return this; } @@ -123,6 +141,7 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) { operation: "select", filters: [], inFilters: [], + overlapsFilters: [], maybeSingle: false, single: false, }; @@ -447,6 +466,149 @@ describe("private document API access", () => { expect(client.calls).toHaveLength(1); }); + it("allows owners to rename the document display title without changing file provenance", async () => { + const original = { + id: documentId, + owner_id: userId, + title: "Original title", + file_name: "source.pdf", + storage_path: `${userId}/documents/${documentId}/source.pdf`, + content_hash: "hash-1", + metadata: { source_path: "C:\\Guidelines\\source.pdf" }, + }; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") return ok(original); + if (call.table === "documents" && call.operation === "update") { + return ok({ + ...original, + ...(call.updatePayload as Record), + }); + } + return ok([]); + }); + mockRuntime(client); + const { PATCH } = await import("../src/app/api/documents/[id]/route"); + + const response = await PATCH( + authenticatedRequest(`/api/documents/${documentId}`, { + method: "PATCH", + body: JSON.stringify({ title: "Better clinical title" }), + }), + { params: Promise.resolve({ id: documentId }) }, + ); + const body = await payload(response); + const updateCall = client.calls.find((call) => call.table === "documents" && call.operation === "update"); + const update = updateCall?.updatePayload as Record; + + expect(response.status).toBe(200); + expect((body.document as Record).title).toBe("Better clinical title"); + expect(update.title).toBe("Better clinical title"); + expect(update).not.toHaveProperty("file_name"); + expect(update).not.toHaveProperty("storage_path"); + expect(update).not.toHaveProperty("content_hash"); + expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId }); + }); + + it("rejects invalid document rename titles", async () => { + const client = createSupabaseMock(); + mockRuntime(client); + const { PATCH } = await import("../src/app/api/documents/[id]/route"); + + const response = await PATCH( + authenticatedRequest(`/api/documents/${documentId}`, { + method: "PATCH", + body: JSON.stringify({ title: " " }), + }), + { params: Promise.resolve({ id: documentId }) }, + ); + + expect(response.status).toBe(400); + expect(await payload(response)).toEqual({ error: "Enter a document title between 1 and 180 characters." }); + expect(client.from).not.toHaveBeenCalled(); + }); + + it("does not rename another user's document", async () => { + const client = createSupabaseMock(() => ok(null)); + mockRuntime(client); + const { PATCH } = await import("../src/app/api/documents/[id]/route"); + + const response = await PATCH( + authenticatedRequest(`/api/documents/${otherDocumentId}`, { + method: "PATCH", + body: JSON.stringify({ title: "Not mine" }), + }), + { params: Promise.resolve({ id: otherDocumentId }) }, + ); + + expect(response.status).toBe(404); + expect(await payload(response)).toEqual({ error: "Document not found." }); + expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId }); + }); + + it("permanently deletes an owned document, query logs, and storage objects", async () => { + const sourcePath = `${userId}/documents/${documentId}/source.pdf`; + const imagePath = `${userId}/images/${imageId}.png`; + const chunkId = "44444444-4444-4444-8444-444444444444"; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") { + return ok({ id: documentId, owner_id: userId, title: "Owned", storage_path: sourcePath }); + } + if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); + if (call.table === "document_images" && call.operation === "select") return ok([{ storage_path: imagePath }]); + if (call.table === "document_chunks" && call.operation === "select") return ok([{ id: chunkId }]); + if (call.table === "rag_queries" && call.operation === "delete") return ok([]); + if (call.table === "documents" && call.operation === "delete") return ok([]); + return ok([]); + }); + mockRuntime(client); + const { DELETE } = await import("../src/app/api/documents/[id]/route"); + + const response = await DELETE( + authenticatedRequest(`/api/documents/${documentId}`, { method: "DELETE" }), + { params: Promise.resolve({ id: documentId }) }, + ); + const body = await payload(response); + const ragDelete = client.calls.find((call) => call.table === "rag_queries" && call.operation === "delete"); + const documentDelete = client.calls.find((call) => call.table === "documents" && call.operation === "delete"); + + expect(response.status).toBe(200); + expect(body).toMatchObject({ deleted: true, documentId, storageWarnings: [] }); + expect(ragDelete?.filters).toContainEqual({ column: "owner_id", value: userId }); + expect(ragDelete?.overlapsFilters).toContainEqual({ column: "source_chunk_ids", values: [chunkId] }); + expect(documentDelete?.filters).toContainEqual({ column: "id", value: documentId }); + expect(documentDelete?.filters).toContainEqual({ column: "owner_id", value: userId }); + expect(client.storageMocks.storageFrom).toHaveBeenCalledWith("clinical-documents"); + expect(client.storageMocks.storageFrom).toHaveBeenCalledWith("clinical-images"); + expect(client.storageMocks.remove).toHaveBeenCalledWith([sourcePath]); + expect(client.storageMocks.remove).toHaveBeenCalledWith([imagePath]); + }); + + it("blocks permanent delete while a document is actively indexing", async () => { + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") { + return ok({ id: documentId, owner_id: userId, title: "Owned", storage_path: "source.pdf" }); + } + if (call.table === "ingestion_jobs" && call.operation === "select") { + return ok([{ id: "job-1", status: "processing" }]); + } + return ok([]); + }); + mockRuntime(client); + const { DELETE } = await import("../src/app/api/documents/[id]/route"); + + const response = await DELETE( + authenticatedRequest(`/api/documents/${documentId}`, { method: "DELETE" }), + { params: Promise.resolve({ id: documentId }) }, + ); + + expect(response.status).toBe(409); + expect(await payload(response)).toEqual({ + error: "Document is currently indexing. Stop or wait for the worker before deleting.", + }); + expect(client.calls.some((call) => call.table === "documents" && call.operation === "delete")).toBe(false); + expect(client.storageMocks.remove).not.toHaveBeenCalled(); + }); + it("passes owner scope through search and answer routes", async () => { const searchChunksWithTelemetry = vi.fn(async () => ({ results: [], From dddab4e2a8e9c1e9173fae1214f8e70ee0bb079f Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 28 May 2026 19:01:09 +0800 Subject: [PATCH 4/7] Harden database access and document cleanup --- package.json | 2 + scripts/check-indexing.ts | 80 ++++++++++++ scripts/cleanup-storage.ts | 116 +++++++++++++++++ scripts/purge-query-logs.ts | 64 +++++++++ src/app/api/documents/[id]/route.ts | 98 +++++++++++++- src/app/api/documents/route.ts | 123 ++++++++++++++++-- src/components/ClinicalDashboard.tsx | 79 ++++++++++- ...07000_database_hardening_before_import.sql | 89 +++++++++++++ supabase/schema.sql | 76 ++++++++--- tests/private-access-routes.test.ts | 29 +++++ tests/supabase-schema.test.ts | 20 ++- tests/ui-smoke.spec.ts | 25 +++- 12 files changed, 764 insertions(+), 37 deletions(-) create mode 100644 scripts/cleanup-storage.ts create mode 100644 scripts/purge-query-logs.ts create mode 100644 supabase/migrations/20260528007000_database_hardening_before_import.sql diff --git a/package.json b/package.json index aefa51570..83f2d4816 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,8 @@ "check:indexing": "tsx scripts/check-indexing.ts", "eval:rag": "tsx scripts/eval-rag.ts", "eval:search": "tsx scripts/eval-search.ts", + "cleanup:storage": "tsx scripts/cleanup-storage.ts", + "purge:query-logs": "tsx scripts/purge-query-logs.ts", "samples": "tsx scripts/generate-sample-documents.ts", "samples:check": "tsx scripts/check-sample-extraction.ts" }, diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index 7efdeb6ee..b2cf1cd71 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -23,11 +23,91 @@ async function main() { if (batchError) throw new Error(batchError.message); const { error: jobError } = await supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts").limit(1); if (jobError) throw new Error(jobError.message); + const { error: cleanupTableError } = await supabase.from("storage_cleanup_jobs").select("id,status").limit(1); + if (cleanupTableError) throw new Error(cleanupTableError.message); + + const { data: documents, error: documentsError } = await supabase + .from("documents") + .select("id,owner_id,title,content_hash,status,page_count,chunk_count"); + if (documentsError) throw new Error(documentsError.message); + + const duplicateHashGroups = new Map(); + for (const document of documents ?? []) { + if (!document.owner_id || !document.content_hash) continue; + const key = `${document.owner_id}:${document.content_hash}`; + duplicateHashGroups.set(key, [...(duplicateHashGroups.get(key) ?? []), document.title ?? document.id]); + } + const duplicateGroups = Array.from(duplicateHashGroups.values()).filter((titles) => titles.length > 1); + const emptyIndexedDocuments = (documents ?? []).filter( + (document) => document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0), + ); + + const [ + missingEmbeddingResult, + failedJobsResult, + activeJobsResult, + imageCountResult, + searchableImageResult, + oversizedBatchesResult, + cleanupIssuesResult, + ] = await Promise.all([ + supabase.from("document_chunks").select("id", { count: "exact", head: true }).is("embedding", null), + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"), + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]), + supabase.from("document_images").select("id", { count: "exact", head: true }), + supabase.from("document_images").select("id", { count: "exact", head: true }).eq("searchable", true), + supabase + .from("import_batches") + .select("id,name,total_files,status") + .gt("total_files", 150) + .in("status", ["queued", "processing"]), + supabase + .from("storage_cleanup_jobs") + .select("id,status,last_error") + .in("status", ["pending", "failed"]) + .order("created_at", { ascending: true }) + .limit(5), + ]); + + for (const result of [ + missingEmbeddingResult, + failedJobsResult, + activeJobsResult, + imageCountResult, + searchableImageResult, + oversizedBatchesResult, + cleanupIssuesResult, + ]) { + if (result.error) throw new Error(result.error.message); + } + + const issues: string[] = []; + if (duplicateGroups.length > 0) issues.push(`duplicate content-hash groups: ${duplicateGroups.length}`); + if ((missingEmbeddingResult.count ?? 0) > 0) issues.push(`chunks missing embeddings: ${missingEmbeddingResult.count}`); + if (emptyIndexedDocuments.length > 0) issues.push(`empty indexed documents: ${emptyIndexedDocuments.length}`); + if ((failedJobsResult.count ?? 0) > 0) issues.push(`failed ingestion jobs: ${failedJobsResult.count}`); + if ((oversizedBatchesResult.data ?? []).length > 0) { + issues.push(`active oversized import batches: ${(oversizedBatchesResult.data ?? []).length}`); + } + if ((cleanupIssuesResult.data ?? []).length > 0) { + issues.push(`pending or failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); + } console.log("Indexing prerequisites ready."); console.log("Supabase bulk ingestion tables are reachable."); console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`); console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`); + console.log(`Documents: ${(documents ?? []).length}; empty indexed: ${emptyIndexedDocuments.length}`); + console.log(`Duplicate content-hash groups: ${duplicateGroups.length}`); + console.log(`Chunks missing embeddings: ${missingEmbeddingResult.count ?? 0}`); + console.log(`Failed jobs: ${failedJobsResult.count ?? 0}; pending/processing jobs: ${activeJobsResult.count ?? 0}`); + console.log(`Images: ${imageCountResult.count ?? 0}; searchable: ${searchableImageResult.count ?? 0}`); + console.log(`Active oversized batches: ${(oversizedBatchesResult.data ?? []).length}`); + console.log(`Pending/failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); + + if (issues.length > 0) { + throw new Error(`Indexing readiness check failed: ${issues.join("; ")}`); + } } main().catch((error) => { diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts new file mode 100644 index 000000000..270852c59 --- /dev/null +++ b/scripts/cleanup-storage.ts @@ -0,0 +1,116 @@ +import { loadEnvConfig } from "@next/env"; +import { loadAdminClient } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +type CleanupArgs = { + limit: number; + dryRun: boolean; +}; + +type CleanupJob = { + id: string; + document_bucket: string | null; + document_paths: string[] | null; + image_bucket: string | null; + image_paths: string[] | null; + attempts: number; +}; + +function parseArgs(argv: string[]): CleanupArgs { + const args: CleanupArgs = { limit: 50, dryRun: false }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--dry-run") { + args.dryRun = true; + continue; + } + if (token === "--limit") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("Missing value for --limit"); + args.limit = Number.parseInt(value, 10); + index += 1; + } + } + + if (!Number.isInteger(args.limit) || args.limit <= 0) throw new Error("--limit must be a positive integer."); + return args; +} + +async function removePaths(args: { supabase: Awaited>; bucket: string; paths: string[] }) { + let removed = 0; + const warnings: string[] = []; + const uniquePaths = Array.from(new Set(args.paths.filter(Boolean))); + + for (let start = 0; start < uniquePaths.length; start += 1000) { + const batch = uniquePaths.slice(start, start + 1000); + const { data, error } = await args.supabase.storage.from(args.bucket).remove(batch); + if (error) { + warnings.push(`${args.bucket}: ${error.message}`); + } else { + removed += data?.length ?? 0; + } + } + + return { removed, warnings }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const supabase = await loadAdminClient(); + const { data, error } = await supabase + .from("storage_cleanup_jobs") + .select("id,document_bucket,document_paths,image_bucket,image_paths,attempts") + .in("status", ["pending", "failed"]) + .order("created_at", { ascending: true }) + .limit(args.limit); + + if (error) throw new Error(error.message); + const jobs = (data ?? []) as CleanupJob[]; + console.log(`Found ${jobs.length} storage cleanup job(s).`); + if (args.dryRun || jobs.length === 0) return; + + let completed = 0; + let failed = 0; + + for (const job of jobs) { + const documentCleanup = await removePaths({ + supabase, + bucket: job.document_bucket ?? "clinical-documents", + paths: job.document_paths ?? [], + }); + const imageCleanup = await removePaths({ + supabase, + bucket: job.image_bucket ?? "clinical-images", + paths: job.image_paths ?? [], + }); + const warnings = [...documentCleanup.warnings, ...imageCleanup.warnings]; + const nextStatus = warnings.length > 0 ? "failed" : "completed"; + const { error: updateError } = await supabase + .from("storage_cleanup_jobs") + .update({ + status: nextStatus, + attempts: job.attempts + 1, + storage_removed: documentCleanup.removed + imageCleanup.removed, + last_error: warnings.length ? warnings.join("; ") : null, + completed_at: nextStatus === "completed" ? new Date().toISOString() : null, + metadata: { + operation: "storage_cleanup_retry", + storage_warnings: warnings, + }, + }) + .eq("id", job.id); + + if (updateError) throw new Error(updateError.message); + if (nextStatus === "completed") completed += 1; + else failed += 1; + } + + console.log(`Storage cleanup complete: ${completed} completed, ${failed} failed.`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/scripts/purge-query-logs.ts b/scripts/purge-query-logs.ts new file mode 100644 index 000000000..fc352989b --- /dev/null +++ b/scripts/purge-query-logs.ts @@ -0,0 +1,64 @@ +import { loadEnvConfig } from "@next/env"; +import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +type PurgeArgs = { + ownerEmail?: string; + olderThanDays: number; + dryRun: boolean; +}; + +function parseArgs(argv: string[]): PurgeArgs { + const args: PurgeArgs = { + ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + olderThanDays: 90, + dryRun: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--dry-run") { + args.dryRun = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + index += 1; + + if (token === "--owner-email") args.ownerEmail = value; + if (token === "--older-than-days") args.olderThanDays = Number.parseInt(value, 10); + } + + if (!args.ownerEmail) throw new Error("Provide --owner-email."); + if (!Number.isInteger(args.olderThanDays) || args.olderThanDays <= 0) { + throw new Error("--older-than-days must be a positive integer."); + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const supabase = await loadAdminClient(); + const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail!); + const before = new Date(Date.now() - args.olderThanDays * 24 * 60 * 60 * 1000).toISOString(); + const countQuery = supabase + .from("rag_queries") + .select("id", { count: "exact", head: true }) + .eq("owner_id", ownerId) + .lt("created_at", before); + const { count, error: countError } = await countQuery; + if (countError) throw new Error(countError.message); + + console.log(`RAG query logs older than ${args.olderThanDays} day(s): ${count ?? 0}`); + if (args.dryRun || !count) return; + + const { error: deleteError } = await supabase.from("rag_queries").delete().eq("owner_id", ownerId).lt("created_at", before); + if (deleteError) throw new Error(deleteError.message); + console.log(`Deleted ${count} old RAG query log(s).`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/app/api/documents/[id]/route.ts b/src/app/api/documents/[id]/route.ts index e59dab3b5..9a53707f5 100644 --- a/src/app/api/documents/[id]/route.ts +++ b/src/app/api/documents/[id]/route.ts @@ -53,6 +53,62 @@ async function removeStorageObjects(args: { return { storageRemoved, storageWarnings: warnings }; } +async function createStorageCleanupJob(args: { + supabase: ReturnType; + ownerId: string; + documentId: string; + documentTitle: string; + sourcePath: string | null; + imagePaths: string[]; +}) { + const { data, error } = await args.supabase + .from("storage_cleanup_jobs") + .insert({ + owner_id: args.ownerId, + document_id: args.documentId, + document_title: args.documentTitle, + document_bucket: env.SUPABASE_DOCUMENT_BUCKET, + document_paths: args.sourcePath ? [args.sourcePath] : [], + image_bucket: env.SUPABASE_IMAGE_BUCKET, + image_paths: Array.from(new Set(args.imagePaths.filter(Boolean))), + status: "pending", + metadata: { + operation: "permanent_document_delete", + created_by: "api/documents/[id]", + }, + }) + .select("id") + .single(); + + if (error) throw new Error(error.message); + return data.id as string; +} + +async function updateStorageCleanupJob(args: { + supabase: ReturnType; + cleanupJobId: string; + status: "completed" | "failed"; + storageRemoved: number; + warnings: string[]; +}) { + const { error } = await args.supabase + .from("storage_cleanup_jobs") + .update({ + status: args.status, + attempts: 1, + storage_removed: args.storageRemoved, + last_error: args.warnings.length ? args.warnings.join("; ") : null, + completed_at: args.status === "completed" ? new Date().toISOString() : null, + metadata: { + operation: "permanent_document_delete", + storage_warnings: args.warnings, + }, + }) + .eq("id", args.cleanupJobId); + + return error ? storageWarningsFrom(error, "Cleanup ledger") : null; +} + export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { try { const { id } = await params; @@ -231,23 +287,59 @@ export async function DELETE(request: Request, { params }: { params: Promise<{ i if (chunksError) throw new Error(chunksError.message); const chunkIds = (chunks ?? []).map((chunk) => chunk.id).filter(Boolean); + const imagePaths = (images ?? []).map((image) => image.storage_path).filter(Boolean); + const cleanupJobId = await createStorageCleanupJob({ + supabase, + ownerId: user.id, + documentId: id, + documentTitle: document.title, + sourcePath: document.storage_path, + imagePaths, + }); + if (chunkIds.length > 0) { const { error: queryDeleteError } = await supabase .from("rag_queries") .delete() .eq("owner_id", user.id) .overlaps("source_chunk_ids", chunkIds); - if (queryDeleteError) throw new Error(queryDeleteError.message); + if (queryDeleteError) { + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: "failed", + storageRemoved: 0, + warnings: [`Query log delete: ${queryDeleteError.message}`], + }); + throw new Error(ledgerWarning ? `${queryDeleteError.message}; ${ledgerWarning}` : queryDeleteError.message); + } } const { error: deleteError } = await supabase.from("documents").delete().eq("id", id).eq("owner_id", user.id); - if (deleteError) throw new Error(deleteError.message); + if (deleteError) { + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: "failed", + storageRemoved: 0, + warnings: [`Database delete: ${deleteError.message}`], + }); + throw new Error(ledgerWarning ? `${deleteError.message}; ${ledgerWarning}` : deleteError.message); + } const cleanup = await removeStorageObjects({ supabase, sourcePath: document.storage_path, - imagePaths: (images ?? []).map((image) => image.storage_path).filter(Boolean), + imagePaths, + }); + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: cleanup.storageWarnings.length > 0 ? "failed" : "completed", + storageRemoved: cleanup.storageRemoved, + warnings: cleanup.storageWarnings, }); + if (ledgerWarning) cleanup.storageWarnings.push(ledgerWarning); invalidateRagCachesForOwner(user.id); return NextResponse.json({ diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c88fb2898..95d993429 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -7,41 +7,145 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f export const runtime = "nodejs"; +const DOCUMENT_LIST_COLUMNS = [ + "id", + "owner_id", + "title", + "description", + "file_name", + "file_type", + "file_size", + "storage_path", + "content_hash", + "source_path", + "import_batch_id", + "status", + "page_count", + "chunk_count", + "image_count", + "error_message", + "metadata", + "created_at", + "updated_at", +].join(","); + +const LABEL_LIST_COLUMNS = [ + "id", + "document_id", + "owner_id", + "label", + "label_type", + "source", + "confidence", + "metadata", + "created_at", + "updated_at", +].join(","); + +const SUMMARY_LIST_COLUMNS = [ + "id", + "document_id", + "owner_id", + "summary", + "clinical_specifics", + "source_chunk_ids", + "source_image_ids", + "model", + "metadata", + "generated_at", + "created_at", + "updated_at", +].join(","); + +const VALID_STATUSES = new Set(["queued", "processing", "indexed", "failed"]); +type DocumentListRow = Record & { id: string }; +type LabelListRow = Record & { document_id: string }; +type SummaryListRow = Record & { document_id: string }; + +function parsePositiveInt(value: string | null, fallback: number, max: number) { + const parsed = Number.parseInt(value ?? "", 10); + if (!Number.isInteger(parsed) || parsed <= 0) return fallback; + return Math.min(parsed, max); +} + +function parseOffset(value: string | null) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 0; +} + +function ilikePattern(value: string) { + return `%${value.replace(/[%_]/g, "\\$&")}%`; +} + +function safeSearchTerm(value: string) { + return value.replace(/[,%()]/g, " ").replace(/\s+/g, " ").trim().slice(0, 120); +} + export async function GET(request: Request) { try { if (isDemoMode()) { return NextResponse.json({ documents: demoDocuments, demoMode: true }); } + const url = new URL(request.url); + const limit = parsePositiveInt(url.searchParams.get("limit"), 100, 200); + const offset = parseOffset(url.searchParams.get("offset")); + const search = safeSearchTerm(url.searchParams.get("q") ?? ""); + const status = url.searchParams.get("status")?.trim() ?? ""; + const includeMeta = url.searchParams.get("includeMeta") !== "false"; + const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); - const { data, error } = await supabase + let query = supabase .from("documents") - .select("*") + .select(DOCUMENT_LIST_COLUMNS, { count: "exact" }) .eq("owner_id", user.id) - .order("created_at", { ascending: false }); + .order("created_at", { ascending: false }) + .range(offset, offset + limit - 1); + + if (status && VALID_STATUSES.has(status)) { + query = query.eq("status", status); + } + if (search) { + const pattern = ilikePattern(search); + query = query.or(`title.ilike.${pattern},file_name.ilike.${pattern}`); + } + + const { data, error, count } = await query; if (error) throw new Error(error.message); - const documents = data ?? []; + const documents = (data ?? []) as unknown as DocumentListRow[]; const documentIds = documents.map((document) => document.id); - if (documentIds.length === 0) return NextResponse.json({ documents }); + const pagination = { + limit, + offset, + total: count ?? documents.length, + nextOffset: offset + documents.length, + hasMore: count === null ? documents.length === limit : offset + documents.length < count, + }; + + if (documentIds.length === 0 || !includeMeta) { + return NextResponse.json({ documents, pagination }); + } const [labelsResult, summariesResult] = await Promise.all([ - supabase.from("document_labels").select("*").in("document_id", documentIds), - supabase.from("document_summaries").select("*").in("document_id", documentIds), + supabase.from("document_labels").select(LABEL_LIST_COLUMNS).in("document_id", documentIds), + supabase.from("document_summaries").select(SUMMARY_LIST_COLUMNS).in("document_id", documentIds), ]); if (labelsResult.error) throw new Error(labelsResult.error.message); if (summariesResult.error) throw new Error(summariesResult.error.message); const labelsByDocument = new Map(); - for (const label of labelsResult.data ?? []) { + for (const label of (labelsResult.data ?? []) as unknown as LabelListRow[]) { const existing = labelsByDocument.get(label.document_id) ?? []; existing.push(label); labelsByDocument.set(label.document_id, existing); } - const summariesByDocument = new Map((summariesResult.data ?? []).map((summary) => [summary.document_id, summary])); + const summariesByDocument = new Map( + ((summariesResult.data ?? []) as unknown as SummaryListRow[]).map((summary) => [summary.document_id, summary]), + ); return NextResponse.json({ documents: documents.map((document) => ({ @@ -49,6 +153,7 @@ export async function GET(request: Request) { labels: labelsByDocument.get(document.id) ?? [], summary: summariesByDocument.get(document.id) ?? null, })), + pagination, }); } catch (error) { if (error instanceof AuthenticationError) { diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index b959b1486..fd4d1d25d 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -115,6 +115,7 @@ const navigationHashes = ["#search", "#quotes", "#images", "#sources"] as const; const themeStorageKey = "clinical-kb-theme"; const themeChangeEvent = "clinical-kb-theme-change"; +const documentPageSize = 150; type SetupCheckStatus = "ready" | "needs_setup" | "unknown"; type SetupCheck = { @@ -123,6 +124,13 @@ type SetupCheck = { status: SetupCheckStatus; detail: string; }; +type DocumentPagination = { + limit: number; + offset: number; + total: number; + nextOffset: number; + hasMore: boolean; +}; type AnswerPayload = RagAnswer & { demoMode?: boolean }; @@ -1441,15 +1449,21 @@ function AuthPanel() { function DocumentDrawer({ documents, + pagination, + loadingMoreDocuments, selectedDocumentIds, onToggleScope, + onLoadMoreDocuments, onDocumentRenamed, onDocumentDeleted, canManageDocuments, }: { documents: ClinicalDocument[]; + pagination: DocumentPagination | null; + loadingMoreDocuments: boolean; selectedDocumentIds: string[]; onToggleScope: (documentId: string) => void; + onLoadMoreDocuments: () => void; onDocumentRenamed: (document: ClinicalDocument) => void; onDocumentDeleted: (result: DocumentDeleteResult) => void; canManageDocuments: boolean; @@ -1473,6 +1487,11 @@ function DocumentDrawer({ className={fieldControlWithIcon} /> + {pagination && pagination.total > documents.length ? ( +

+ Showing {documents.length} of {pagination.total} documents. Load more to manage older files. +

+ ) : null}
{filtered.length === 0 ? ( + {pagination?.hasMore ? ( + + ) : null}
); } @@ -2057,6 +2087,8 @@ export function ClinicalDashboard() { const scrollFrameRef = useRef(null); const navSyncLockRef = useRef(null); const [documents, setDocuments] = useState([]); + const [documentsPagination, setDocumentsPagination] = useState(null); + const [loadingMoreDocuments, setLoadingMoreDocuments] = useState(false); const [jobs, setJobs] = useState([]); const [batches, setBatches] = useState([]); const [query, setQuery] = useState(""); @@ -2103,6 +2135,7 @@ export function ClinicalDashboard() { if (nextDemoMode) setDemoMode(true); if (!nextDemoMode && authStatus !== "authenticated") { setDocuments([]); + setDocumentsPagination(null); setJobs([]); setBatches([]); return; @@ -2111,7 +2144,7 @@ export function ClinicalDashboard() { const protectedHeaders = nextDemoMode ? undefined : authorizationHeader; const [documentsResponse, jobsResponse, batchesResponse] = await Promise.all([ - fetch("/api/documents", { headers: protectedHeaders }), + fetch(`/api/documents?limit=${documentPageSize}`, { headers: protectedHeaders }), fetch("/api/ingestion/jobs", { headers: protectedHeaders }), fetch("/api/ingestion/batches", { headers: protectedHeaders }), ]); @@ -2119,6 +2152,7 @@ export function ClinicalDashboard() { if (documentsResponse.status === 401 || jobsResponse.status === 401 || batchesResponse.status === 401) { markSessionExpired(); setDocuments([]); + setDocumentsPagination(null); setJobs([]); setBatches([]); return; @@ -2127,6 +2161,7 @@ export function ClinicalDashboard() { if (documentsResponse.ok) { const payload = await documentsResponse.json(); setDocuments(payload.documents ?? []); + setDocumentsPagination(payload.pagination ?? null); if (payload.demoMode) setDemoMode(true); if (payload.setupRequired) setSetupWarning(payload.error); } else { @@ -2151,6 +2186,45 @@ export function ClinicalDashboard() { } }, [authStatus, authorizationHeader, clientDemoMode, markSessionExpired]); + const loadMoreDocuments = useCallback(async () => { + if (!documentsPagination?.hasMore || loadingMoreDocuments || (!clientDemoMode && authStatus !== "authenticated")) { + return; + } + + setLoadingMoreDocuments(true); + try { + const protectedHeaders = clientDemoMode ? undefined : authorizationHeader; + const response = await fetch( + `/api/documents?limit=${documentPageSize}&offset=${documentsPagination.nextOffset}`, + { headers: protectedHeaders }, + ); + if (response.status === 401) { + markSessionExpired(); + return; + } + if (!response.ok) { + setApiUnavailable(true); + return; + } + const payload = await response.json(); + const nextDocuments = (payload.documents ?? []) as ClinicalDocument[]; + setDocuments((current) => { + const seen = new Set(current.map((document) => document.id)); + return [...current, ...nextDocuments.filter((document) => !seen.has(document.id))]; + }); + setDocumentsPagination(payload.pagination ?? null); + } finally { + setLoadingMoreDocuments(false); + } + }, [ + authStatus, + authorizationHeader, + clientDemoMode, + documentsPagination, + loadingMoreDocuments, + markSessionExpired, + ]); + const retryJob = useCallback( async (jobId: string) => { setIndexingActionId(jobId); @@ -2687,8 +2761,11 @@ export function ClinicalDashboard() { > { return this; } + range(from: number, to: number) { + this.call.range = { from, to }; + return this; + } + + or(filter: string) { + this.call.orFilters.push(filter); + return this; + } + limit(count: number) { this.call.limitCount = count; return this; @@ -139,6 +151,7 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) { const call: QueryCall = { table, operation: "select", + orFilters: [], filters: [], inFilters: [], overlapsFilters: [], @@ -230,7 +243,11 @@ describe("private document API access", () => { expect(response.status).toBe(200); expect(body.documents).toEqual(documents.map((document) => ({ ...document, labels: [], summary: null }))); + expect(body.pagination).toMatchObject({ limit: 100, offset: 0, nextOffset: 1, hasMore: false }); expect(client.calls[0].filters).toContainEqual({ column: "owner_id", value: userId }); + expect(client.calls[0].selected).toContain("id,owner_id,title"); + expect(client.calls[0].selected).not.toBe("*"); + expect(client.calls[0].range).toEqual({ from: 0, to: 99 }); }); it("does not return raw internal database errors", async () => { @@ -556,6 +573,8 @@ describe("private document API access", () => { if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); if (call.table === "document_images" && call.operation === "select") return ok([{ storage_path: imagePath }]); if (call.table === "document_chunks" && call.operation === "select") return ok([{ id: chunkId }]); + if (call.table === "storage_cleanup_jobs" && call.operation === "insert") return ok({ id: "cleanup-1" }); + if (call.table === "storage_cleanup_jobs" && call.operation === "update") return ok([]); if (call.table === "rag_queries" && call.operation === "delete") return ok([]); if (call.table === "documents" && call.operation === "delete") return ok([]); return ok([]); @@ -570,9 +589,19 @@ describe("private document API access", () => { const body = await payload(response); const ragDelete = client.calls.find((call) => call.table === "rag_queries" && call.operation === "delete"); const documentDelete = client.calls.find((call) => call.table === "documents" && call.operation === "delete"); + const cleanupInsert = client.calls.find((call) => call.table === "storage_cleanup_jobs" && call.operation === "insert"); + const cleanupUpdate = client.calls.find((call) => call.table === "storage_cleanup_jobs" && call.operation === "update"); expect(response.status).toBe(200); expect(body).toMatchObject({ deleted: true, documentId, storageWarnings: [] }); + expect(cleanupInsert?.insertPayload).toMatchObject({ + owner_id: userId, + document_id: documentId, + document_paths: [sourcePath], + image_paths: [imagePath], + status: "pending", + }); + expect(cleanupUpdate?.updatePayload).toMatchObject({ status: "completed", storage_removed: 0 }); expect(ragDelete?.filters).toContainEqual({ column: "owner_id", value: userId }); expect(ragDelete?.overlapsFilters).toContainEqual({ column: "source_chunk_ids", values: [chunkId] }); expect(documentDelete?.filters).toContainEqual({ column: "id", value: documentId }); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index c6c876116..15793789b 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -8,14 +8,17 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("public.import_batches,"); expect(schema).toContain("public.document_labels,"); expect(schema).toContain("public.document_summaries,"); - expect(schema).toMatch(/grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries to service_role;/); + expect(schema).toContain("public.storage_cleanup_jobs"); + expect(schema).toMatch(/grant select, insert, update, delete on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.image_caption_cache, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to service_role;/); expect(schema).toContain("grant execute on all functions in schema public to service_role;"); }); - it("keeps authenticated direct access aligned with RLS policies", () => { - expect(schema).toContain("grant select, insert, update, delete on table public.documents to authenticated;"); - expect(schema).toMatch(/grant select on table .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs to authenticated;/); - expect(schema).toContain("grant select, insert on table public.rag_queries to authenticated;"); + it("keeps browser Data API grants read-only except manual labels", () => { + expect(schema).toContain("revoke all privileges on all tables in schema public from anon, authenticated;"); + expect(schema).toContain("revoke execute on all functions in schema public from public, anon, authenticated;"); + expect(schema).toMatch(/grant select on table .*public\.documents, .*public\.document_pages, .*public\.document_images, .*public\.document_labels, .*public\.document_summaries, .*public\.document_chunks, .*public\.ingestion_jobs, .*public\.rag_queries, .*public\.storage_cleanup_jobs to authenticated;/); + expect(schema).not.toContain("grant select, insert, update, delete on table public.documents to authenticated;"); + expect(schema).not.toContain("grant select, insert on table public.rag_queries to authenticated;"); expect(schema).not.toMatch(/grant .* on table .* to anon;/); }); @@ -30,6 +33,13 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("create or replace function public.reset_document_index"); }); + it("tracks retryable storage cleanup and query-log purge performance", () => { + expect(schema).toContain("create table if not exists public.storage_cleanup_jobs"); + expect(schema).toContain("create index if not exists storage_cleanup_jobs_owner_status_idx"); + expect(schema).toContain("create index if not exists rag_queries_source_chunk_ids_gin_idx"); + expect(schema).toContain("create policy \"storage cleanup owner read\""); + }); + it("filters hybrid retrieval by owner inside Postgres", () => { expect(schema).toContain("owner_filter uuid default null"); expect(schema).toContain("and (owner_filter is null or d.owner_id = owner_filter)"); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index 2ceaa904e..1ae806455 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -72,8 +72,13 @@ async function mockPrivateAuthenticatedApi(page: Page) { json: { demoMode: false, checks: readySetupChecks }, }); }); - await page.route(/\/api\/documents$/, async (route) => { - await route.fulfill({ json: { documents: [] } }); + await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => { + await route.fulfill({ + json: { + documents: [], + pagination: { limit: 150, offset: 0, total: 0, nextOffset: 0, hasMore: false }, + }, + }); }); await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => { await route.fulfill({ json: { jobs: [] } }); @@ -148,8 +153,20 @@ async function mockDemoApi(page: Page) { json: { demoMode: true, checks: readySetupChecks }, }); }); - await page.route(/\/api\/documents$/, async (route) => { - await route.fulfill({ json: { documents: demoDocuments, demoMode: true } }); + await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => { + await route.fulfill({ + json: { + documents: demoDocuments, + demoMode: true, + pagination: { + limit: 150, + offset: 0, + total: demoDocuments.length, + nextOffset: demoDocuments.length, + hasMore: false, + }, + }, + }); }); await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => { await route.fulfill({ json: { jobs: [], demoMode: true } }); From 4a65c1a8ba774fb9e73cbb3a66f000e360a603c3 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 1 Jun 2026 01:31:48 +0800 Subject: [PATCH 5/7] Fix dashboard compile and browser overlay regressions --- .env.example | 13 +- package.json | 1 + scripts/audit-tables.ts | 188 +++ scripts/check-indexing.ts | 185 ++- scripts/cleanup-storage.ts | 6 +- scripts/enrich-documents.ts | 406 ++++- scripts/eval-rag.ts | 23 +- scripts/eval-search.ts | 45 +- scripts/eval-utils.ts | 3 +- scripts/import-documents.ts | 10 +- scripts/purge-query-logs.ts | 6 +- src/app/api/answer/route.ts | 34 +- src/app/api/answer/stream/route.ts | 77 +- src/app/api/documents/[id]/reindex/route.ts | 26 +- src/app/api/documents/[id]/route.ts | 57 +- src/app/api/documents/route.ts | 6 +- src/app/api/search/route.ts | 94 +- src/app/api/setup-status/route.ts | 12 +- src/app/layout.tsx | 12 - src/components/AccessibleTable.tsx | 85 + src/components/ClinicalDashboard.tsx | 1012 ++++++++++-- src/components/DocumentManagementActions.tsx | 4 +- src/components/DocumentViewer.tsx | 165 +- src/lib/answer-ranking.ts | 273 ++++ src/lib/bulk-import.ts | 10 +- src/lib/chunking.ts | 169 +- src/lib/clinical-search.ts | 405 ++++- src/lib/deep-memory.ts | 536 +++++++ src/lib/document-enrichment.ts | 142 +- src/lib/env.ts | 23 +- src/lib/evidence.ts | 139 +- src/lib/extractors/document.ts | 15 +- src/lib/http.ts | 11 +- src/lib/image-filtering.ts | 197 +++ src/lib/openai.ts | 133 +- src/lib/public-rate-limit.ts | 63 + src/lib/rag-eval-cases.ts | 40 + src/lib/rag-routing.ts | 61 +- src/lib/rag.ts | 1393 ++++++++++++++--- src/lib/supabase/auth.ts | 73 + src/lib/supabase/client.tsx | 111 +- src/lib/supabase/project.ts | 4 +- src/lib/types.ts | 136 +- .../20260528008000_table_visual_metadata.sql | 37 + .../20260528009000_deep_memory_indexing.sql | 147 ++ supabase/schema.sql | 123 ++ tests/answer-ranking.test.ts | 163 ++ tests/chunking.test.ts | 19 + tests/clinical-search.test.ts | 171 +- tests/deep-memory.test.ts | 201 +++ tests/document-enrichment.test.ts | 170 ++ tests/eval-search.test.ts | 36 + tests/evidence.test.ts | 87 + tests/image-filtering.test.ts | 86 + tests/openai-cache.test.ts | 16 +- tests/pdf-extractor.test.ts | 142 ++ tests/private-access-routes.test.ts | 344 +++- tests/rag-answer-fallback.test.ts | 197 +++ tests/rag-routing.test.ts | 34 +- tests/rag-trust.test.ts | 118 +- tests/supabase-schema.test.ts | 46 +- tests/ui-smoke.spec.ts | 103 +- worker/main.ts | 242 ++- worker/python/extract_pdf_assets.py | 525 ++++++- 64 files changed, 8674 insertions(+), 737 deletions(-) create mode 100644 scripts/audit-tables.ts create mode 100644 src/components/AccessibleTable.tsx create mode 100644 src/lib/answer-ranking.ts create mode 100644 src/lib/deep-memory.ts create mode 100644 src/lib/public-rate-limit.ts create mode 100644 supabase/migrations/20260528008000_table_visual_metadata.sql create mode 100644 supabase/migrations/20260528009000_deep_memory_indexing.sql create mode 100644 tests/answer-ranking.test.ts create mode 100644 tests/deep-memory.test.ts create mode 100644 tests/document-enrichment.test.ts create mode 100644 tests/eval-search.test.ts create mode 100644 tests/pdf-extractor.test.ts create mode 100644 tests/rag-answer-fallback.test.ts diff --git a/.env.example b/.env.example index d5e826e49..dadd4d6ae 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,17 @@ SUPABASE_PROJECT_NAME=Clinical KB Database NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=your-publishable-or-anon-key SUPABASE_SERVICE_ROLE_KEY=your-service-role-key +# Local-only no-auth mode (development only). +# Enable one or both of these to load real data without per-request browser auth: +# - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server) +# - LOCAL_NO_AUTH (server only) +#NEXT_PUBLIC_LOCAL_NO_AUTH=false +#LOCAL_NO_AUTH=false + +# Optional local no-auth owner identity. Set one of these in local dev. +#LOCAL_NO_AUTH_OWNER_ID= +#LOCAL_NO_AUTH_OWNER_EMAIL= + # OpenAI direct API. This app sends extracted guideline text and extracted images # to OpenAI for embeddings, captioning, and grounded answer generation. OPENAI_API_KEY=replace-with-openai-api-key @@ -14,7 +25,7 @@ OPENAI_EMBEDDING_MODEL=text-embedding-3-small OPENAI_ANSWER_MODEL=gpt-5.4-mini OPENAI_FAST_ANSWER_MODEL=gpt-5.4-mini OPENAI_STRONG_ANSWER_MODEL=gpt-5.4 -OPENAI_MAX_OUTPUT_TOKENS=900 +OPENAI_MAX_OUTPUT_TOKENS=1400 OPENAI_QUERY_CACHE_SIZE=200 OPENAI_VISION_MODEL=gpt-5.4-mini OPENAI_REQUEST_TIMEOUT_MS=45000 diff --git a/package.json b/package.json index 83f2d4816..547b235f6 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "eval:search": "tsx scripts/eval-search.ts", "cleanup:storage": "tsx scripts/cleanup-storage.ts", "purge:query-logs": "tsx scripts/purge-query-logs.ts", + "audit:tables": "tsx scripts/audit-tables.ts", "samples": "tsx scripts/generate-sample-documents.ts", "samples:check": "tsx scripts/check-sample-extraction.ts" }, diff --git a/scripts/audit-tables.ts b/scripts/audit-tables.ts new file mode 100644 index 000000000..b6a20d883 --- /dev/null +++ b/scripts/audit-tables.ts @@ -0,0 +1,188 @@ +import { loadEnvConfig } from "@next/env"; +import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils"; +import { assessClinicalImageUse } from "@/lib/image-filtering"; + +loadEnvConfig(process.cwd()); + +type Args = { + ownerEmail?: string; + allOwners: boolean; + document?: string; + limit: number; + failOnMissed: boolean; +}; + +function parseArgs(): Args { + const args: Args = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, allOwners: !process.env.RAG_EVAL_OWNER_EMAIL, limit: 200, failOnMissed: false }; + const tokens = process.argv.slice(2); + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + const value = tokens[index + 1]; + if (token === "--owner-email") { + args.ownerEmail = value; + args.allOwners = false; + index += 1; + } else if (token === "--all-owners") { + args.allOwners = true; + } else if (token === "--document") { + args.document = value; + index += 1; + } else if (token === "--limit") { + args.limit = Math.max(1, Math.min(Number(value) || 200, 1000)); + index += 1; + } else if (token === "--fail-on-missed") { + args.failOnMissed = true; + } + } + return args; +} + +function textSuggestsTable(text: string) { + return /\b(table\s+\d+[a-z]?|appendix\s+\d+[a-z]?|roles?\s+and\s+responsibilities|score\b.*\bmanagement|observation|medication|dose|frequency)\b/i.test( + text, + ); +} + +function compactTitle(title: string) { + return title.length > 72 ? `${title.slice(0, 69).trim()}...` : title; +} + +async function main() { + const args = parseArgs(); + if (!args.ownerEmail && !args.allOwners) throw new Error('Provide --owner-email "you@example.com" or --all-owners.'); + + const supabase = await loadAdminClient(); + const ownerId = args.ownerEmail && !args.allOwners ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + + let documentQuery = supabase + .from("documents") + .select("id,title,file_name,status,page_count,image_count") + .order("created_at", { ascending: true }) + .limit(args.limit); + if (ownerId) documentQuery = documentQuery.eq("owner_id", ownerId); + if (args.document) { + documentQuery = documentQuery.or( + `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, + ); + } + + const { data: documents, error: documentsError } = await documentQuery; + if (documentsError) throw new Error(documentsError.message); + const documentIds = (documents ?? []).map((document) => document.id as string); + if (documentIds.length === 0) { + console.log("No documents matched the audit filters."); + return; + } + + const [imagesResult, pagesResult] = await Promise.all([ + supabase + .from("document_images") + .select("document_id,source_kind,searchable,image_type,clinical_relevance_score,metadata") + .in("document_id", documentIds) + .neq("image_type", "logo_decorative"), + supabase.from("document_pages").select("document_id,page_number,text").in("document_id", documentIds), + ]); + if (imagesResult.error) throw new Error(imagesResult.error.message); + if (pagesResult.error) throw new Error(pagesResult.error.message); + + const counts = new Map(); + for (const documentId of documentIds) { + counts.set(documentId, { tables: 0, clinicalTables: 0, adminTables: 0, searchableAdminTables: 0, images: 0 }); + } + for (const image of imagesResult.data ?? []) { + const documentId = String(image.document_id); + const current = counts.get(documentId) ?? { + tables: 0, + clinicalTables: 0, + adminTables: 0, + searchableAdminTables: 0, + images: 0, + }; + const metadata = image.metadata && typeof image.metadata === "object" ? (image.metadata as Record) : {}; + const useClass = String( + metadata.clinical_use_class ?? + assessClinicalImageUse({ + imageType: image.image_type, + searchable: image.searchable, + clinicalRelevanceScore: image.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: typeof metadata.table_role === "string" ? metadata.table_role : null, + tableText: + typeof metadata.table_text === "string" + ? metadata.table_text + : typeof metadata.table_text_snippet === "string" + ? metadata.table_text_snippet + : null, + }).clinical_use_class, + ); + if (image.searchable !== false) current.images += 1; + if (image.source_kind === "table_crop") { + current.tables += 1; + if (useClass === "clinical_evidence" && image.searchable !== false) current.clinicalTables += 1; + if (["administrative", "reference"].includes(useClass)) current.adminTables += 1; + if (["administrative", "reference"].includes(useClass) && image.searchable !== false) { + current.searchableAdminTables += 1; + } + } + counts.set(documentId, current); + } + + const tableMarkerPages = new Map>(); + for (const page of pagesResult.data ?? []) { + if (!textSuggestsTable(String(page.text ?? ""))) continue; + const documentId = String(page.document_id); + const pages = tableMarkerPages.get(documentId) ?? new Set(); + const pageNumber = Number(page.page_number); + if (Number.isFinite(pageNumber)) pages.add(pageNumber); + tableMarkerPages.set(documentId, pages); + } + + const possibleMisses: string[] = []; + const searchableAdminIssues: string[] = []; + console.log(`Table audit for ${documents?.length ?? 0} documents`); + for (const document of documents ?? []) { + const documentCounts = counts.get(document.id) ?? { + tables: 0, + clinicalTables: 0, + adminTables: 0, + searchableAdminTables: 0, + images: 0, + }; + const markerPages = Array.from(tableMarkerPages.get(document.id) ?? []).sort((a, b) => a - b); + if (markerPages.length > 0 && documentCounts.tables === 0) { + possibleMisses.push(`${document.file_name}: table-like text on pages ${markerPages.slice(0, 8).join(", ")}`); + } + if (documentCounts.searchableAdminTables > 0) { + searchableAdminIssues.push(`${document.file_name}: searchable admin/reference tables=${documentCounts.searchableAdminTables}`); + } + console.log( + [ + compactTitle(document.file_name ?? document.title ?? document.id), + `status=${document.status}`, + `pages=${document.page_count ?? 0}`, + `tables=${documentCounts.tables}`, + `clinicalTables=${documentCounts.clinicalTables}`, + `adminReferenceTables=${documentCounts.adminTables}`, + `searchableAdminReference=${documentCounts.searchableAdminTables}`, + `searchableImages=${documentCounts.images}`, + markerPages.length ? `tableTextPages=${markerPages.slice(0, 8).join(",")}` : "tableTextPages=none", + ].join(" | "), + ); + } + + console.log(`Possible missed table documents: ${possibleMisses.length}`); + for (const item of possibleMisses.slice(0, 20)) console.log(`- ${item}`); + console.log(`Searchable admin/reference table issues: ${searchableAdminIssues.length}`); + for (const item of searchableAdminIssues.slice(0, 20)) console.log(`- ${item}`); + if (searchableAdminIssues.length > 0) { + throw new Error("Table audit found admin/reference tables still marked searchable."); + } + if (args.failOnMissed && possibleMisses.length > 0) { + throw new Error("Table audit found documents with table-like text but no retained table crops."); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index b2cf1cd71..2c0ec9818 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -2,14 +2,89 @@ import { loadEnvConfig } from "@next/env"; loadEnvConfig(process.cwd()); -async function main() { - const [{ env, requireOpenAIEnv, requireServerEnv }, { createAdminClient }, { checkPythonPdfPrerequisites }] = - await Promise.all([ - import("@/lib/env"), - import("@/lib/supabase/admin"), - import("../worker/prerequisites"), +type MetadataRow = { document_id: string; metadata?: unknown; source?: string | null }; + +function metadataRecord(metadata: unknown): Record { + return metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? (metadata as Record) + : {}; +} + +function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) { + return metadataRecord(metadata).rag_enrichment_version === expectedVersion; +} + +function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) { + const record = metadataRecord(metadata); + return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion; +} + +function strictEnrichmentVersionRequired() { + return process.argv.includes("--strict-enrichment-version") || process.env.RAG_REQUIRE_CURRENT_ENRICHMENT_VERSION === "1"; +} + +async function loadEnrichmentRows(supabase: any, documentIds: string[]) { + const summaries: MetadataRow[] = []; + const labels: MetadataRow[] = []; + + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + const [summaryResult, labelResult] = await Promise.all([ + supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids), + supabase.from("document_labels").select("document_id,source,metadata").in("document_id", ids), ]); + if (summaryResult.error) throw new Error(summaryResult.error.message); + if (labelResult.error) throw new Error(labelResult.error.message); + summaries.push(...((summaryResult.data ?? []) as MetadataRow[])); + labels.push(...((labelResult.data ?? []) as MetadataRow[])); + } + + return { summaries, labels }; +} + +async function loadRowsForDocuments(supabase: any, table: string, select: string, documentIds: string[]) { + const rows: MetadataRow[] = []; + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + for (let rangeStart = 0; ; rangeStart += 1000) { + const { data, error } = await supabase + .from(table) + .select(select) + .in("document_id", ids) + .range(rangeStart, rangeStart + 999); + if (error) throw new Error(error.message); + rows.push(...((data ?? []) as MetadataRow[])); + if (!data || data.length < 1000) break; + } + } + return rows; +} + +async function loadDeepMemoryRows(supabase: any, documentIds: string[]) { + const [sections, memoryCards, chunks] = await Promise.all([ + loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_chunks", "document_id,metadata", documentIds), + ]); + return { sections, memoryCards, chunks }; +} + +async function main() { + const [ + { env, requireOpenAIEnv, requireServerEnv }, + { createAdminClient }, + { checkPythonPdfPrerequisites }, + { ragEnrichmentVersion }, + { ragDeepMemoryVersion }, + ] = await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("../worker/prerequisites"), + import("@/lib/document-enrichment"), + import("@/lib/deep-memory"), + ]); + requireServerEnv(); requireOpenAIEnv(); @@ -28,9 +103,36 @@ async function main() { const { data: documents, error: documentsError } = await supabase .from("documents") - .select("id,owner_id,title,content_hash,status,page_count,chunk_count"); + .select("id,owner_id,title,content_hash,status,page_count,chunk_count,metadata"); if (documentsError) throw new Error(documentsError.message); + const indexedDocuments = (documents ?? []).filter((document) => document.status === "indexed"); + const enrichmentRows = await loadEnrichmentRows( + supabase, + indexedDocuments.map((document) => document.id), + ); + const deepMemoryRows = await loadDeepMemoryRows( + supabase, + indexedDocuments.map((document) => document.id), + ); + const summariesByDocument = new Map(enrichmentRows.summaries.map((row) => [row.document_id, row])); + const labelRowsByDocument = new Map(); + const sectionRowsByDocument = new Map(); + const memoryRowsByDocument = new Map(); + const chunkRowsByDocument = new Map(); + for (const label of enrichmentRows.labels) { + labelRowsByDocument.set(label.document_id, [...(labelRowsByDocument.get(label.document_id) ?? []), label]); + } + for (const section of deepMemoryRows.sections) { + sectionRowsByDocument.set(section.document_id, [...(sectionRowsByDocument.get(section.document_id) ?? []), section]); + } + for (const card of deepMemoryRows.memoryCards) { + memoryRowsByDocument.set(card.document_id, [...(memoryRowsByDocument.get(card.document_id) ?? []), card]); + } + for (const chunk of deepMemoryRows.chunks) { + chunkRowsByDocument.set(chunk.document_id, [...(chunkRowsByDocument.get(chunk.document_id) ?? []), chunk]); + } + const duplicateHashGroups = new Map(); for (const document of documents ?? []) { if (!document.owner_id || !document.content_hash) continue; @@ -39,8 +141,45 @@ async function main() { } const duplicateGroups = Array.from(duplicateHashGroups.values()).filter((titles) => titles.length > 1); const emptyIndexedDocuments = (documents ?? []).filter( - (document) => document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0), + (document) => + document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0), + ); + const documentsMissingSummaries = indexedDocuments.filter((document) => !summariesByDocument.has(document.id)); + const documentsMissingGeneratedLabels = indexedDocuments.filter((document) => + (labelRowsByDocument.get(document.id) ?? []).every((label) => label.source !== "generated"), + ); + const documentsWithCurrentEnrichmentVersion = indexedDocuments.filter((document) => { + const summary = summariesByDocument.get(document.id); + const generatedLabels = (labelRowsByDocument.get(document.id) ?? []).filter((label) => label.source === "generated"); + return ( + hasCurrentEnrichmentVersion(document.metadata, ragEnrichmentVersion) && + hasCurrentEnrichmentVersion(summary?.metadata, ragEnrichmentVersion) && + generatedLabels.some((label) => hasCurrentEnrichmentVersion(label.metadata, ragEnrichmentVersion)) + ); + }); + const documentsMissingCurrentEnrichmentVersion = + indexedDocuments.length - documentsWithCurrentEnrichmentVersion.length; + const documentsMissingSections = indexedDocuments.filter( + (document) => (sectionRowsByDocument.get(document.id) ?? []).length === 0, ); + const documentsMissingMemoryCards = indexedDocuments.filter( + (document) => (memoryRowsByDocument.get(document.id) ?? []).length === 0, + ); + const documentsWithCurrentDeepMemoryVersion = indexedDocuments.filter((document) => { + const sections = sectionRowsByDocument.get(document.id) ?? []; + const memoryCards = memoryRowsByDocument.get(document.id) ?? []; + const chunks = chunkRowsByDocument.get(document.id) ?? []; + return ( + hasCurrentMemoryVersion(document.metadata, ragDeepMemoryVersion) && + sections.some((section) => hasCurrentMemoryVersion(section.metadata, ragDeepMemoryVersion)) && + memoryCards.some((card) => hasCurrentMemoryVersion(card.metadata, ragDeepMemoryVersion)) && + chunks.length > 0 && + chunks.every((chunk) => hasCurrentMemoryVersion(chunk.metadata, ragDeepMemoryVersion)) + ); + }); + const documentsMissingCurrentDeepMemoryVersion = + indexedDocuments.length - documentsWithCurrentDeepMemoryVersion.length; + const requireCurrentEnrichmentVersion = strictEnrichmentVersionRequired(); const [ missingEmbeddingResult, @@ -53,7 +192,10 @@ async function main() { ] = await Promise.all([ supabase.from("document_chunks").select("id", { count: "exact", head: true }).is("embedding", null), supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"), - supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]), + supabase + .from("ingestion_jobs") + .select("id", { count: "exact", head: true }) + .in("status", ["pending", "processing"]), supabase.from("document_images").select("id", { count: "exact", head: true }), supabase.from("document_images").select("id", { count: "exact", head: true }).eq("searchable", true), supabase @@ -83,8 +225,22 @@ async function main() { const issues: string[] = []; if (duplicateGroups.length > 0) issues.push(`duplicate content-hash groups: ${duplicateGroups.length}`); - if ((missingEmbeddingResult.count ?? 0) > 0) issues.push(`chunks missing embeddings: ${missingEmbeddingResult.count}`); + if ((missingEmbeddingResult.count ?? 0) > 0) + issues.push(`chunks missing embeddings: ${missingEmbeddingResult.count}`); if (emptyIndexedDocuments.length > 0) issues.push(`empty indexed documents: ${emptyIndexedDocuments.length}`); + if (documentsMissingSummaries.length > 0) + issues.push(`indexed documents missing summaries: ${documentsMissingSummaries.length}`); + if (documentsMissingGeneratedLabels.length > 0) + issues.push(`indexed documents missing generated labels: ${documentsMissingGeneratedLabels.length}`); + if (documentsMissingSections.length > 0) issues.push(`indexed documents missing sections: ${documentsMissingSections.length}`); + if (documentsMissingMemoryCards.length > 0) + issues.push(`indexed documents missing memory cards: ${documentsMissingMemoryCards.length}`); + if (requireCurrentEnrichmentVersion && documentsMissingCurrentEnrichmentVersion > 0) { + issues.push(`indexed documents missing current enrichment version: ${documentsMissingCurrentEnrichmentVersion}`); + } + if (requireCurrentEnrichmentVersion && documentsMissingCurrentDeepMemoryVersion > 0) { + issues.push(`indexed documents missing current deep-memory version: ${documentsMissingCurrentDeepMemoryVersion}`); + } if ((failedJobsResult.count ?? 0) > 0) issues.push(`failed ingestion jobs: ${failedJobsResult.count}`); if ((oversizedBatchesResult.data ?? []).length > 0) { issues.push(`active oversized import batches: ${(oversizedBatchesResult.data ?? []).length}`); @@ -98,6 +254,15 @@ async function main() { console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`); console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`); console.log(`Documents: ${(documents ?? []).length}; empty indexed: ${emptyIndexedDocuments.length}`); + console.log( + `Enrichment coverage: summaries missing=${documentsMissingSummaries.length}; generated labels missing=${documentsMissingGeneratedLabels.length}`, + ); + console.log( + `RAG enrichment version: ${documentsWithCurrentEnrichmentVersion.length}/${indexedDocuments.length} indexed current (${ragEnrichmentVersion}); strict=${requireCurrentEnrichmentVersion ? "yes" : "no"}`, + ); + console.log( + `Deep memory version: ${documentsWithCurrentDeepMemoryVersion.length}/${indexedDocuments.length} indexed current (${ragDeepMemoryVersion}); sections missing=${documentsMissingSections.length}; memory cards missing=${documentsMissingMemoryCards.length}`, + ); console.log(`Duplicate content-hash groups: ${duplicateGroups.length}`); console.log(`Chunks missing embeddings: ${missingEmbeddingResult.count ?? 0}`); console.log(`Failed jobs: ${failedJobsResult.count ?? 0}; pending/processing jobs: ${activeJobsResult.count ?? 0}`); diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts index 270852c59..27389bc13 100644 --- a/scripts/cleanup-storage.ts +++ b/scripts/cleanup-storage.ts @@ -38,7 +38,11 @@ function parseArgs(argv: string[]): CleanupArgs { return args; } -async function removePaths(args: { supabase: Awaited>; bucket: string; paths: string[] }) { +async function removePaths(args: { + supabase: Awaited>; + bucket: string; + paths: string[]; +}) { let removed = 0; const warnings: string[] = []; const uniquePaths = Array.from(new Set(args.paths.filter(Boolean))); diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index 35cbd065e..0642ed266 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -5,12 +5,15 @@ loadEnvConfig(process.cwd()); type EnrichArgs = { ownerEmail?: string; + allOwners: boolean; mode: string; limit: number; documentId?: string; + includeCurrent: boolean; }; type SupabaseAdmin = Awaited>; +type MetadataRow = { id?: string; document_id: string; metadata?: unknown; source?: string | null }; async function loadAdminClient() { const { createAdminClient } = await import("@/lib/supabase/admin"); @@ -20,13 +23,23 @@ async function loadAdminClient() { function parseArgs(argv: string[]): EnrichArgs { const args: EnrichArgs = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + allOwners: !process.env.RAG_EVAL_OWNER_EMAIL, mode: "summaries-labels-images", limit: 25, + includeCurrent: false, }; for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; if (!token.startsWith("--")) continue; + if (token === "--all-owners") { + args.allOwners = true; + continue; + } + if (token === "--include-current") { + args.includeCurrent = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); index += 1; @@ -56,15 +69,15 @@ async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) { throw new Error(`No Supabase Auth user found for ${email}. Sign in once before enriching documents.`); } -async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId: string) { +async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId?: string) { let query = supabase .from("documents") .select("id,owner_id,title,file_name,source_path,status,metadata") - .eq("owner_id", ownerId) .eq("status", "indexed") .order("created_at", { ascending: true }) - .limit(args.limit); + .limit(args.documentId ? 1 : Math.max(args.limit * 10, 1000)); + if (ownerId) query = query.eq("owner_id", ownerId); if (args.documentId) query = query.eq("id", args.documentId); const { data, error } = await query; @@ -72,21 +85,88 @@ async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId: return data ?? []; } +async function loadEnrichmentCoverage(supabase: SupabaseAdmin, documentIds: string[]) { + const summaries: MetadataRow[] = []; + const labels: MetadataRow[] = []; + + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + const [summaryResult, labelResult] = await Promise.all([ + supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids), + supabase.from("document_labels").select("id,document_id,source,metadata").in("document_id", ids), + ]); + + if (summaryResult.error) throw new Error(summaryResult.error.message); + if (labelResult.error) throw new Error(labelResult.error.message); + summaries.push(...((summaryResult.data ?? []) as MetadataRow[])); + labels.push(...((labelResult.data ?? []) as MetadataRow[])); + } + + const coverage = new Map(); + for (const documentId of documentIds) coverage.set(documentId, { labels: [] }); + for (const summary of summaries) { + coverage.set(summary.document_id, { ...(coverage.get(summary.document_id) ?? { labels: [] }), summary }); + } + for (const label of labels) { + const existing = coverage.get(label.document_id) ?? { labels: [] }; + coverage.set(label.document_id, { ...existing, labels: [...existing.labels, label] }); + } + + return coverage; +} + +async function loadRowsForDocuments(supabase: SupabaseAdmin, table: string, select: string, documentIds: string[]) { + const rows: MetadataRow[] = []; + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + for (let rangeStart = 0; ; rangeStart += 1000) { + const { data, error } = await supabase + .from(table) + .select(select) + .in("document_id", ids) + .range(rangeStart, rangeStart + 999); + if (error) throw new Error(error.message); + rows.push(...((data ?? []) as unknown as MetadataRow[])); + if (!data || data.length < 1000) break; + } + } + return rows; +} + +async function loadDeepMemoryCoverage(supabase: SupabaseAdmin, documentIds: string[]) { + const [sections, memoryCards] = await Promise.all([ + loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds), + ]); + + const coverage = new Map(); + for (const documentId of documentIds) coverage.set(documentId, { sections: [], memoryCards: [] }); + for (const section of sections) { + const existing = coverage.get(section.document_id) ?? { sections: [], memoryCards: [] }; + coverage.set(section.document_id, { ...existing, sections: [...existing.sections, section] }); + } + for (const card of memoryCards) { + const existing = coverage.get(card.document_id) ?? { sections: [], memoryCards: [] }; + coverage.set(card.document_id, { ...existing, memoryCards: [...existing.memoryCards, card] }); + } + return coverage; +} + async function loadEvidence(supabase: SupabaseAdmin, documentId: string) { const [chunksResult, imagesResult] = await Promise.all([ supabase .from("document_chunks") - .select("id,page_number,chunk_index,section_heading,content") + .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata") .eq("document_id", documentId) .order("chunk_index", { ascending: true }) - .limit(24), + .limit(1000), supabase .from("document_images") - .select("id,page_number,caption,image_type,labels") + .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata") .eq("document_id", documentId) .eq("searchable", true) .order("clinical_relevance_score", { ascending: false }) - .limit(12), + .limit(200), ]); if (chunksResult.error) throw new Error(chunksResult.error.message); @@ -98,13 +178,67 @@ function hashBytes(bytes: Buffer) { return createHash("sha256").update(bytes).digest("hex"); } +function metadataString(metadata: unknown, key: string) { + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return null; + const value = (metadata as Record)[key]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function metadataRecord(metadata: unknown): Record { + return metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? { ...(metadata as Record) } + : {}; +} + +function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) { + return metadataRecord(metadata).rag_enrichment_version === expectedVersion; +} + +function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) { + const record = metadataRecord(metadata); + return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion; +} + +function needsEnrichmentBackfill(args: { + document: { metadata?: unknown }; + coverage?: { summary?: MetadataRow; labels: MetadataRow[] }; + memoryCoverage?: { sections: MetadataRow[]; memoryCards: MetadataRow[] }; + ragEnrichmentVersion: string; + ragDeepMemoryVersion: string; +}) { + const generatedLabels = args.coverage?.labels.filter((label) => label.source === "generated") ?? []; + const sections = args.memoryCoverage?.sections ?? []; + const memoryCards = args.memoryCoverage?.memoryCards ?? []; + return ( + !args.coverage?.summary || + generatedLabels.length === 0 || + !hasCurrentEnrichmentVersion(args.document.metadata, args.ragEnrichmentVersion) || + !hasCurrentEnrichmentVersion(args.coverage.summary?.metadata, args.ragEnrichmentVersion) || + generatedLabels.every((label) => !hasCurrentEnrichmentVersion(label.metadata, args.ragEnrichmentVersion)) || + sections.length === 0 || + memoryCards.length === 0 || + !hasCurrentMemoryVersion(args.document.metadata, args.ragDeepMemoryVersion) || + sections.every((section) => !hasCurrentMemoryVersion(section.metadata, args.ragDeepMemoryVersion)) || + memoryCards.every((card) => !hasCurrentMemoryVersion(card.metadata, args.ragDeepMemoryVersion)) + ); +} + async function classifyExistingImages(supabase: SupabaseAdmin, documentId: string) { - const [{ env }, { cheapImageSkipReason, classifiedImageSkipReason, lightweightPerceptualHash }, { classifyAndCaptionImageFromBase64 }] = - await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]); + const [ + { env }, + { + assessClinicalImageUse, + cheapImageSkipReason, + classifiedImageSkipReason, + clinicalImagePolicyVersion, + lightweightPerceptualHash, + }, + { classifyAndCaptionImageFromBase64 }, + ] = await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]); const { data: images, error } = await supabase .from("document_images") .select( - "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason", + "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason,labels,metadata", ) .eq("document_id", documentId) .order("page_number", { ascending: true }); @@ -116,11 +250,8 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin let skipped = 0; for (const image of images ?? []) { - if ( - (image.image_type && image.image_type !== "unclear") || - image.clinical_relevance_score > 0 || - image.skip_reason - ) { + const existingMetadata = metadataRecord(image.metadata); + if (existingMetadata.image_policy_version === clinicalImagePolicyVersion) { if (image.searchable) searchable += 1; else skipped += 1; continue; @@ -147,7 +278,13 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin bbox: image.bbox as [number, number, number, number] | null, width: image.width, height: image.height, - sourceKind: image.source_kind as "embedded" | "diagram_crop" | "page_region" | "fallback" | undefined, + sourceKind: image.source_kind as + | "embedded" + | "table_crop" + | "diagram_crop" + | "page_region" + | "fallback" + | undefined, }, }); @@ -167,70 +304,231 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin } seenHashes.add(imageHash); - const classification = await classifyAndCaptionImageFromBase64({ - base64: bytes.toString("base64"), - mimeType: image.mime_type, - nearbyText: image.caption ?? undefined, + const baseAssessment = assessClinicalImageUse({ + imageType: image.image_type, + searchable: image.searchable, + clinicalRelevanceScore: image.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + tableTitle: metadataString(image.metadata, "table_title"), + tableLabel: metadataString(image.metadata, "table_label"), + caption: image.caption, + labels: Array.isArray(image.labels) ? image.labels : [], + skipReason: image.skip_reason, + }); + const classification = + image.source_kind === "table_crop" && ["administrative", "reference"].includes(baseAssessment.clinical_use_class) + ? { + image_type: image.image_type || "clinical_table", + searchable: false, + clinical_relevance_score: 0, + labels: [], + caption: + baseAssessment.clinical_use_class === "administrative" + ? "Administrative document-control table retained for audit, not clinical evidence." + : "Reference table retained for audit, not clinical evidence.", + skip_reason: baseAssessment.clinical_use_reason, + clinical_use_class: baseAssessment.clinical_use_class, + clinical_use_reason: baseAssessment.clinical_use_reason, + clinical_signal_score: baseAssessment.clinical_signal_score, + admin_signal_score: baseAssessment.admin_signal_score, + } + : await classifyAndCaptionImageFromBase64({ + base64: bytes.toString("base64"), + mimeType: image.mime_type, + nearbyText: image.caption ?? undefined, + sourceKind: image.source_kind, + candidateType: metadataString(image.metadata, "candidate_type"), + tableLabel: metadataString(image.metadata, "table_label"), + tableTitle: metadataString(image.metadata, "table_title"), + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + }); + const finalAssessment = assessClinicalImageUse({ + imageType: classification.image_type, + searchable: classification.searchable, + clinicalRelevanceScore: classification.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + tableTitle: metadataString(image.metadata, "table_title"), + tableLabel: metadataString(image.metadata, "table_label"), + caption: classification.caption, + labels: classification.labels, + skipReason: classification.skip_reason, + }); + const classifiedSkip = classifiedImageSkipReason({ + ...classification, + searchable: finalAssessment.searchable, + clinical_relevance_score: finalAssessment.clinical_relevance_score, + clinical_use_class: finalAssessment.clinical_use_class, + clinical_use_reason: finalAssessment.clinical_use_reason, + clinical_signal_score: finalAssessment.clinical_signal_score, + admin_signal_score: finalAssessment.admin_signal_score, }); - const classifiedSkip = classifiedImageSkipReason(classification); + const retainAsAuditTable = + image.source_kind === "table_crop" && + ["administrative", "reference"].includes(finalAssessment.clinical_use_class) && + classification.image_type !== "logo_decorative"; + const nextSearchable = finalAssessment.searchable; await supabase .from("document_images") .update({ caption: classification.caption || image.caption, image_type: classification.image_type, - searchable: !classifiedSkip, - clinical_relevance_score: classifiedSkip ? 0 : classification.clinical_relevance_score, - skip_reason: classifiedSkip, + searchable: nextSearchable, + clinical_relevance_score: nextSearchable ? finalAssessment.clinical_relevance_score : 0, + skip_reason: nextSearchable ? null : classifiedSkip, image_hash: imageHash, perceptual_hash: perceptualHash, labels: classification.labels, + metadata: { + ...metadataRecord(image.metadata), + clinical_use_class: finalAssessment.clinical_use_class, + clinical_use_reason: finalAssessment.clinical_use_reason, + clinical_signal_score: finalAssessment.clinical_signal_score, + admin_signal_score: finalAssessment.admin_signal_score, + image_policy_version: clinicalImagePolicyVersion, + retained_for_audit: retainAsAuditTable || undefined, + retained_for_document_view: retainAsAuditTable || undefined, + }, }) .eq("id", image.id); - if (classifiedSkip) skipped += 1; + if (!nextSearchable) skipped += 1; else searchable += 1; } return { searchable, skipped, total: images?.length ?? 0 }; } +async function stampExistingEnrichmentVersion(args: { + supabase: SupabaseAdmin; + document: { id: string; metadata?: unknown }; + coverage: { summary?: MetadataRow; labels: MetadataRow[] }; + ragEnrichmentVersion: string; +}) { + const stampedAt = new Date().toISOString(); + const marker = { + rag_enrichment_version: args.ragEnrichmentVersion, + rag_enrichment_updated_at: stampedAt, + version_stamped_at: stampedAt, + }; + + const { error: documentError } = await args.supabase + .from("documents") + .update({ metadata: { ...metadataRecord(args.document.metadata), ...marker } }) + .eq("id", args.document.id); + if (documentError) throw new Error(documentError.message); + + if (args.coverage.summary) { + const { error: summaryError } = await args.supabase + .from("document_summaries") + .update({ metadata: { ...metadataRecord(args.coverage.summary.metadata), ...marker } }) + .eq("document_id", args.document.id); + if (summaryError) throw new Error(summaryError.message); + } + + for (const label of args.coverage.labels.filter((item) => item.source === "generated")) { + if (!label.id) continue; + const { error: labelError } = await args.supabase + .from("document_labels") + .update({ metadata: { ...metadataRecord(label.metadata), ...marker } }) + .eq("id", label.id); + if (labelError) throw new Error(labelError.message); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) { - throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); + if (!args.ownerEmail && !args.allOwners) { + throw new Error('Provide --owner-email "you@example.com", set RAG_EVAL_OWNER_EMAIL, or pass --all-owners.'); } - if (args.mode !== "summaries-labels-images") { - throw new Error("--mode currently supports summaries-labels-images."); + if (!["summaries-labels-images", "metadata-stamp", "deep-memory"].includes(args.mode)) { + throw new Error("--mode supports summaries-labels-images, deep-memory, or metadata-stamp."); } - const [{ requireOpenAIEnv, requireServerEnv }, { upsertDocumentEnrichment }, supabase] = await Promise.all([ + const [ + { requireOpenAIEnv, requireServerEnv }, + { ragEnrichmentVersion, upsertDocumentEnrichment }, + { ragDeepMemoryVersion, upsertDocumentDeepMemory }, + supabase, + ] = await Promise.all([ import("@/lib/env"), import("@/lib/document-enrichment"), + import("@/lib/deep-memory"), loadAdminClient(), ]); requireServerEnv(); - requireOpenAIEnv(); + if (args.mode === "summaries-labels-images" || args.mode === "deep-memory") requireOpenAIEnv(); + + const ownerId = args.ownerEmail && !args.allOwners ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + const loadedDocuments = await loadDocuments(supabase, args, ownerId); + const coverage = await loadEnrichmentCoverage( + supabase, + loadedDocuments.map((document) => document.id), + ); + const memoryCoverage = await loadDeepMemoryCoverage( + supabase, + loadedDocuments.map((document) => document.id), + ); + const documents = loadedDocuments + .filter((document) => + args.includeCurrent + ? true + : needsEnrichmentBackfill({ + document, + coverage: coverage.get(document.id), + memoryCoverage: memoryCoverage.get(document.id), + ragEnrichmentVersion, + ragDeepMemoryVersion, + }), + ) + .slice(0, args.limit); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); - const documents = await loadDocuments(supabase, args, ownerId); - console.log(`Enriching ${documents.length} indexed document(s).`); + console.log( + `Enriching ${documents.length} indexed document(s). scope=${ownerId ? "owner" : "all"} version=${ragEnrichmentVersion}`, + ); let completed = 0; for (const document of documents) { - const imageStats = await classifyExistingImages(supabase, document.id); - await supabase - .from("documents") - .update({ - image_count: imageStats.searchable, - metadata: { - ...(document.metadata ?? {}), - searchable_image_count: imageStats.searchable, - skipped_image_count: imageStats.skipped, - image_enriched_at: new Date().toISOString(), - }, - }) - .eq("id", document.id); + const documentCoverage = coverage.get(document.id); + if (args.mode === "metadata-stamp") { + if (!documentCoverage?.summary || documentCoverage.labels.every((label) => label.source !== "generated")) { + console.log(`SKIP cannot metadata-stamp missing enrichment: ${document.file_name}`); + continue; + } + await stampExistingEnrichmentVersion({ + supabase, + document, + coverage: documentCoverage, + ragEnrichmentVersion, + }); + completed += 1; + console.log(`STAMPED ${document.file_name}`); + continue; + } + + let imageMetadata = metadataRecord(document.metadata); + let imageStats = { searchable: 0, skipped: 0, total: 0 }; + if (args.mode === "summaries-labels-images") { + imageStats = await classifyExistingImages(supabase, document.id); + imageMetadata = { + ...imageMetadata, + searchable_image_count: imageStats.searchable, + skipped_image_count: imageStats.skipped, + image_enriched_at: new Date().toISOString(), + }; + await supabase + .from("documents") + .update({ + image_count: imageStats.searchable, + metadata: imageMetadata, + }) + .eq("id", document.id); + } const evidence = await loadEvidence(supabase, document.id); if (evidence.chunks.length === 0) { @@ -238,15 +536,23 @@ async function main() { continue; } - await upsertDocumentEnrichment({ + if (args.mode === "summaries-labels-images") { + await upsertDocumentEnrichment({ + supabase, + document: { ...document, metadata: imageMetadata }, + chunks: evidence.chunks, + images: evidence.images, + }); + } + const deepMemory = await upsertDocumentDeepMemory({ supabase, - document, + document: { ...document, metadata: imageMetadata }, chunks: evidence.chunks, images: evidence.images, }); completed += 1; console.log( - `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, + `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} sections=${deepMemory.sections.length} memory=${deepMemory.memoryCards.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, ); } diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index 93cdb1836..3daff4673 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -1,13 +1,7 @@ import { loadEnvConfig } from "@next/env"; import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { RagAnswer } from "@/lib/types"; -import { - estimateCostUsd, - findOwnerIdByEmail, - loadAdminClient, - percentile, - validateRagAnswer, -} from "./eval-utils"; +import { estimateCostUsd, findOwnerIdByEmail, loadAdminClient, percentile, validateRagAnswer } from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -90,7 +84,8 @@ function summarizeFailures(results: EvalResult[]) { const failedCases = results.filter((result) => result.failures.length > 0); const failures: string[] = []; - if (supported.length >= 18 && groundedSupported < 17) failures.push(`supported grounded ${groundedSupported}/${supported.length}`); + if (supported.length >= 18 && groundedSupported < 17) + failures.push(`supported grounded ${groundedSupported}/${supported.length}`); if (unsupported.length >= 2 && unsupportedCorrect !== unsupported.length) { failures.push(`unsupported correct ${unsupportedCorrect}/${unsupported.length}`); } @@ -147,14 +142,15 @@ function printHumanSummary(results: EvalResult[]) { console.log(` token_totals=${JSON.stringify(tokenTotals)}`); console.log( ` estimated_cost_usd=${ - estimatedCostUsd === null ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION" : estimatedCostUsd.toFixed(6) + estimatedCostUsd === null + ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION" + : estimatedCostUsd.toFixed(6) }`, ); } async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); const [{ requireOpenAIEnv, requireServerEnv }, { answerQuestionWithScope }, supabase] = await Promise.all([ import("@/lib/env"), @@ -165,11 +161,12 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); + const ownerId = args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + const scope = ownerId ? `owner:${args.ownerEmail}` : "public"; const cases = selectRagEvalCases({ limit: args.limit, question: args.question }); const results: EvalResult[] = []; - if (!args.json) console.log(`Running ${cases.length} RAG eval case(s).`); + if (!args.json) console.log(`Running ${cases.length} RAG eval case(s), scope=${scope}.`); for (const testCase of cases) { const answer = (await answerQuestionWithScope({ @@ -225,7 +222,7 @@ async function main() { const thresholdFailures = summarizeFailures(results); if (args.json) { - console.log(JSON.stringify({ results, thresholdFailures }, null, 2)); + console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2)); } else { printHumanSummary(results); if (thresholdFailures.length > 0) { diff --git a/scripts/eval-search.ts b/scripts/eval-search.ts index 7c16c6151..f58de3eef 100644 --- a/scripts/eval-search.ts +++ b/scripts/eval-search.ts @@ -1,8 +1,15 @@ import { loadEnvConfig } from "@next/env"; +import { pathToFileURL } from "node:url"; import { buildVisualEvidence } from "@/lib/evidence"; import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { SearchResult } from "@/lib/types"; -import { expectedFileHit, findOwnerIdByEmail, hasInvalidVisualEvidence, loadAdminClient, percentile } from "./eval-utils"; +import { + expectedFileHit, + findOwnerIdByEmail, + hasInvalidVisualEvidence, + loadAdminClient, + percentile, +} from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -14,7 +21,7 @@ type EvalArgs = { failOnThreshold: boolean; }; -type SearchEvalResult = { +export type SearchEvalResult = { id: string; question: string; category: RagEvalCase["category"]; @@ -78,13 +85,14 @@ function validateSearchResult(testCase: RagEvalCase, result: Omit= 0.82) failures.push("unsupported control produced high-confidence match"); + if (!testCase.supported && result.topScore >= 0.82) + failures.push("unsupported control produced high-confidence match"); if (testCase.category === "routine" && result.latencyMs > 1500) failures.push("routine search over 1500ms"); return failures; } -function summarizeFailures(results: SearchEvalResult[]) { +export function summarizeFailures(results: SearchEvalResult[]) { const routine = results.filter((result) => result.category === "routine"); const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); @@ -94,8 +102,10 @@ function summarizeFailures(results: SearchEvalResult[]) { const unsupportedHighConfidence = unsupported.filter((result) => result.topScore >= 0.82).length; const failures: string[] = []; - if (expectedHits < 18) failures.push(`expected document top-3 hit ${expectedHits}/${results.length}`); - if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500) failures.push("routine search p95 over 1500ms"); + if (results.length >= 18 && expectedHits < 18) + failures.push(`expected document top-3 hit ${expectedHits}/${results.length}`); + if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500) + failures.push("routine search p95 over 1500ms"); if (routine.length > 0 && routineEmbeddingSkipped / routine.length < 0.7) { failures.push(`routine embedding skipped ${routineEmbeddingSkipped}/${routine.length}`); } @@ -133,7 +143,6 @@ function printHumanSummary(results: SearchEvalResult[]) { async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); const [{ requireOpenAIEnv, requireServerEnv }, { searchChunksWithTelemetry }, supabase] = await Promise.all([ import("@/lib/env"), @@ -144,11 +153,12 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); + const ownerId = args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + const scope = ownerId ? `owner:${args.ownerEmail}` : "public"; const cases = selectRagEvalCases({ limit: args.limit, question: args.question }); const results: SearchEvalResult[] = []; - if (!args.json) console.log(`Running ${cases.length} search eval case(s).`); + if (!args.json) console.log(`Running ${cases.length} search eval case(s), scope=${scope}.`); for (const testCase of cases) { const startedAt = Date.now(); @@ -160,8 +170,9 @@ async function main() { skipCache: true, }); const latencyMs = - search.telemetry.supabase_rpc_latency_ms + search.telemetry.embedding_latency_ms + search.telemetry.rerank_latency_ms || - Date.now() - startedAt; + search.telemetry.supabase_rpc_latency_ms + + search.telemetry.embedding_latency_ms + + search.telemetry.rerank_latency_ms || Date.now() - startedAt; const visuals = buildVisualEvidence(search.results); const baseResult = { id: testCase.id, @@ -198,7 +209,7 @@ async function main() { const thresholdFailures = summarizeFailures(results); if (args.json) { - console.log(JSON.stringify({ results, thresholdFailures }, null, 2)); + console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2)); } else { printHumanSummary(results); if (thresholdFailures.length > 0) { @@ -209,7 +220,9 @@ async function main() { if (args.failOnThreshold && thresholdFailures.length > 0) process.exit(1); } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); + }); +} diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index fa28faa25..224263738 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -49,7 +49,8 @@ export function validateRagAnswer(testCase: RagEvalCase, answer: RagAnswer) { if (testCase.supported && !answer.grounded) failures.push("expected grounded answer"); if (!testCase.supported && answer.grounded) failures.push("expected unsupported answer"); if (!testCase.allowedRoutes.includes(route)) failures.push(`unexpected route ${route}`); - if (answer.citations.length < testCase.minCitations) failures.push(`expected at least ${testCase.minCitations} citations`); + if (answer.citations.length < testCase.minCitations) + failures.push(`expected at least ${testCase.minCitations} citations`); if (testCase.expectedFiles.length > 0 && !expectedHit) failures.push("expected document not in retrieved sources"); if (testCase.requireVisualEvidence && visualEvidence.length === 0) failures.push("expected visual evidence"); if (hasInvalidVisualEvidence(visualEvidence)) failures.push("decorative or zero-relevance visual evidence returned"); diff --git a/scripts/import-documents.ts b/scripts/import-documents.ts index e57569583..e0efb2e88 100644 --- a/scripts/import-documents.ts +++ b/scripts/import-documents.ts @@ -216,10 +216,12 @@ async function main() { const documentId = createDocumentId(); const storagePath = buildImportStoragePath(ownerId, documentId, file.fileName); - const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, await readFile(file.absolutePath), { - contentType: "application/pdf", - upsert: false, - }); + const upload = await supabase.storage + .from(env.SUPABASE_DOCUMENT_BUCKET) + .upload(storagePath, await readFile(file.absolutePath), { + contentType: "application/pdf", + upsert: false, + }); if (upload.error) throw new Error(upload.error.message); const { error: documentError } = await supabase.from("documents").insert({ diff --git a/scripts/purge-query-logs.ts b/scripts/purge-query-logs.ts index fc352989b..fef3ce764 100644 --- a/scripts/purge-query-logs.ts +++ b/scripts/purge-query-logs.ts @@ -53,7 +53,11 @@ async function main() { console.log(`RAG query logs older than ${args.olderThanDays} day(s): ${count ?? 0}`); if (args.dryRun || !count) return; - const { error: deleteError } = await supabase.from("rag_queries").delete().eq("owner_id", ownerId).lt("created_at", before); + const { error: deleteError } = await supabase + .from("rag_queries") + .delete() + .eq("owner_id", ownerId) + .lt("created_at", before); if (deleteError) throw new Error(deleteError.message); console.log(`Deleted ${count} old RAG query log(s).`); } diff --git a/src/app/api/answer/route.ts b/src/app/api/answer/route.ts index 6c77324bf..e5596c597 100644 --- a/src/app/api/answer/route.ts +++ b/src/app/api/answer/route.ts @@ -3,14 +3,13 @@ import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { answerQuestionWithScope } from "@/lib/rag"; -import { jsonError } from "@/lib/http"; -import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { consumePublicAnswerRateLimit } from "@/lib/public-rate-limit"; export const runtime = "nodejs"; const answerSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), }); @@ -25,19 +24,34 @@ export async function POST(request: Request) { }); } - const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); + const rateLimit = consumePublicAnswerRateLimit(request.headers); + if (rateLimit.limited) { + return NextResponse.json( + { error: "Too many public answer requests. Retry shortly." }, + { status: 429, headers: { "Retry-After": String(rateLimit.retryAfterSeconds) } }, + ); + } + const answer = await answerQuestionWithScope({ query: body.query, documentId: body.documentId, documentIds: body.documentIds, - ownerId: user.id, + ownerId: undefined, }); return NextResponse.json(answer); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof PublicApiError) { + return jsonError(error, error.status); + } + if (error instanceof Error) { + return jsonError( + new PublicApiError("Answer generation failed. Retry with a narrower question.", 500, { code: error.name }), + 500, + ); } - return jsonError(error, 400); + return jsonError("Answer generation failed.", 500); } } diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index 37eeec3c1..8812c30c9 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -2,14 +2,13 @@ import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { PublicApiError, jsonError } from "@/lib/http"; +import { consumePublicAnswerRateLimit, type PublicRateLimitResult } from "@/lib/public-rate-limit"; import { answerQuestionWithScope, type AnswerProgressEvent } from "@/lib/rag"; -import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; const answerSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), }); @@ -20,6 +19,51 @@ function encodeSse(event: string, data: unknown) { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } +function rateLimitStream(rateLimit: PublicRateLimitResult) { + return new Response( + encodeSse("error", { + error: "Too many public answer requests. Retry shortly.", + status: 429, + details: { retryAfterSeconds: rateLimit.retryAfterSeconds, resetAt: rateLimit.resetAt }, + }), + { + status: 429, + headers: { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + "Retry-After": String(rateLimit.retryAfterSeconds), + }, + }, + ); +} + +function streamErrorPayload(error: unknown) { + if (error instanceof PublicApiError) { + return { message: error.message, status: error.status, details: error.details }; + } + + if (error instanceof Error) { + return { + message: "Answer generation failed. Retry with a narrower question.", + status: 503, + details: { code: error.name }, + }; + } + + return { + message: "Search processing is temporarily unavailable.", + status: 503, + }; +} + +function logStreamError(error: unknown) { + console.error("Search stream failed", { + name: error instanceof Error ? error.name : typeof error, + message: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); +} + function streamAnswer(body: AnswerBody, ownerId?: string) { const encoder = new TextEncoder(); @@ -44,11 +88,9 @@ function streamAnswer(body: AnswerBody, ownerId?: string) { }); send("final", answer); } catch (error) { - const publicError = - error instanceof PublicApiError - ? error - : new PublicApiError("Answer generation failed. Retry with a narrower question.", 500); - send("error", { error: publicError.message }); + logStreamError(error); + const streamError = streamErrorPayload(error); + send("error", { error: streamError.message, status: streamError.status, details: streamError.details }); } finally { controller.close(); } @@ -69,13 +111,20 @@ export async function POST(request: Request) { const body = answerSchema.parse(await request.json()); if (isDemoMode()) return streamAnswer(body); - const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); - return streamAnswer(body, user.id); + const rateLimit = consumePublicAnswerRateLimit(request.headers); + if (rateLimit.limited) return rateLimitStream(rateLimit); + + return streamAnswer(body); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof PublicApiError) { + return jsonError(error, error.status); + } + if (error instanceof Error) { + return jsonError(new PublicApiError("Answer processing failed.", 500, { code: error.name }), 500); } - return jsonError(error, 400); + return jsonError("Answer processing failed.", 500); } } diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 7575a4f03..181b8c83c 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { env, isDemoMode } from "@/lib/env"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; +import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { jsonError } from "@/lib/http"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -27,7 +28,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const { data: document, error: documentError } = await supabase .from("documents") - .select("id,owner_id,title,file_name,source_path,import_batch_id") + .select("id,owner_id,title,file_name,source_path,import_batch_id,metadata") .eq("id", id) .eq("owner_id", user.id) .maybeSingle(); @@ -39,17 +40,17 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const [chunksResult, imagesResult] = await Promise.all([ supabase .from("document_chunks") - .select("id,page_number,chunk_index,section_heading,content") + .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata") .eq("document_id", id) .order("chunk_index", { ascending: true }) - .limit(24), + .limit(1000), supabase .from("document_images") - .select("id,page_number,caption,image_type,labels") + .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata") .eq("document_id", id) .eq("searchable", true) .order("clinical_relevance_score", { ascending: false }) - .limit(12), + .limit(200), ]); if (chunksResult.error) throw new Error(chunksResult.error.message); @@ -64,7 +65,20 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: chunks: chunksResult.data, images: imagesResult.data ?? [], }); - return NextResponse.json({ mode, enrichment }); + const deepMemory = await upsertDocumentDeepMemory({ + supabase, + document, + chunks: chunksResult.data, + images: imagesResult.data ?? [], + }); + return NextResponse.json({ + mode, + enrichment, + deepMemory: { + sectionCount: deepMemory.sections.length, + memoryCardCount: deepMemory.memoryCards.length, + }, + }); } const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: id }); diff --git a/src/app/api/documents/[id]/route.ts b/src/app/api/documents/[id]/route.ts index 9a53707f5..aceb7cd7e 100644 --- a/src/app/api/documents/[id]/route.ts +++ b/src/app/api/documents/[id]/route.ts @@ -17,8 +17,56 @@ function safeMetadata(value: unknown) { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } +function metadataText(metadata: Record, key: string) { + const value = metadata[key]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function compactTableText(value: string | null, limit = 500) { + if (!value) return null; + const compact = value.replace(/\s+/g, " ").trim(); + if (!compact) return null; + return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact; +} + +function metadataStringArrayRows(metadata: Record, key: string) { + const value = metadata[key]; + if (!Array.isArray(value)) return null; + const rows = value + .filter((row): row is unknown[] => Array.isArray(row)) + .map((row) => row.map((cell) => String(cell ?? "").trim())); + return rows.length ? rows : null; +} + +function metadataStringArray(metadata: Record, key: string) { + const value = metadata[key]; + if (!Array.isArray(value)) return null; + const items = value.map((item) => String(item ?? "").trim()).filter(Boolean); + return items.length ? items : null; +} + +function withImageTableMetadata(image: T) { + const metadata = safeMetadata(image.metadata); + const tableText = metadataText(metadata, "table_text") ?? metadataText(metadata, "table_text_snippet"); + const publicImage = { ...image }; + delete publicImage.metadata; + return { + ...publicImage, + tableLabel: metadataText(metadata, "table_label"), + tableTitle: metadataText(metadata, "table_title"), + tableRole: metadataText(metadata, "table_role"), + tableTextSnippet: compactTableText(tableText), + clinicalUseClass: metadataText(metadata, "clinical_use_class"), + clinicalUseReason: metadataText(metadata, "clinical_use_reason"), + accessibleTableMarkdown: metadataText(metadata, "accessible_table_markdown"), + tableRows: metadataStringArrayRows(metadata, "table_rows"), + tableColumns: metadataStringArray(metadata, "table_columns"), + }; +} + function storageWarningsFrom(error: unknown, label: string) { - const message = error && typeof error === "object" && "message" in error ? String(error.message) : "Storage cleanup failed."; + const message = + error && typeof error === "object" && "message" in error ? String(error.message) : "Storage cleanup failed."; return `${label}: ${message}`; } @@ -145,10 +193,11 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: const { data: images, error: imagesError } = await supabase .from("document_images") .select( - "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels", + "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata", ) .eq("document_id", id) - .eq("searchable", true) + .neq("image_type", "logo_decorative") + .or("searchable.eq.true,source_kind.eq.table_crop") .order("page_number", { ascending: true }); if (imagesError) throw new Error(imagesError.message); @@ -181,7 +230,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: summary: summaryResult.data ?? null, }, pages: pages ?? [], - images: images ?? [], + images: (images ?? []).map(withImageTableMetadata), chunks: chunks ?? [], }); } catch (error) { diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 95d993429..e3105d154 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -78,7 +78,11 @@ function ilikePattern(value: string) { } function safeSearchTerm(value: string) { - return value.replace(/[,%()]/g, " ").replace(/\s+/g, " ").trim().slice(0, 120); + return value + .replace(/[,%()]/g, " ") + .replace(/\s+/g, " ") + .trim() + .slice(0, 120); } export async function GET(request: Request) { diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 87aefd48d..9b93cc4ec 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -3,22 +3,78 @@ import { z } from "zod"; import { demoSearch } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { buildSmartPanel, buildVisualEvidence, diversifySearchResults } from "@/lib/evidence"; -import { fetchRelatedDocuments } from "@/lib/document-enrichment"; -import { jsonError } from "@/lib/http"; +import { fetchRelatedDocuments, toDocumentMatch } from "@/lib/document-enrichment"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { isClinicalImageEvidence } from "@/lib/image-filtering"; import { searchChunksWithTelemetry } from "@/lib/rag"; import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; const searchSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), topK: z.number().int().min(1).max(20).optional(), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), + mode: z.enum(["answer", "documents"]).optional().default("answer"), + documentLimit: z.number().int().min(1).max(50).optional().default(20), includeRelatedDocuments: z.boolean().optional().default(true), }); +function buildDocumentMatchesFromResults(results: Awaited>, limit: number) { + const grouped = new Map< + string, + { + document_id: string; + title: string; + file_name: string; + bestPages: number[]; + bestChunkIds: string[]; + imageCount: number; + tableCount: number; + score: number; + } + >(); + for (const result of results) { + const current = grouped.get(result.document_id); + const score = result.hybrid_score ?? result.similarity; + const page = result.page_number ?? null; + const clinicalImages = result.images?.filter((image) => isClinicalImageEvidence(image)) ?? []; + const tableCount = clinicalImages.filter((image) => image.source_kind === "table_crop").length; + const imageCount = clinicalImages.length; + if (!current) { + grouped.set(result.document_id, { + document_id: result.document_id, + title: result.title, + file_name: result.file_name, + bestPages: page ? [page] : [], + bestChunkIds: [result.id], + imageCount, + tableCount, + score, + }); + continue; + } + current.score = Math.max(current.score, score); + if (page && !current.bestPages.includes(page)) current.bestPages.push(page); + if (!current.bestChunkIds.includes(result.id)) current.bestChunkIds.push(result.id); + current.imageCount += imageCount; + current.tableCount += tableCount; + } + + return Array.from(grouped.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map((document) => ({ + ...document, + labels: [], + summarySnippet: null, + matchReason: `Matched ${document.bestChunkIds.length} indexed passage${ + document.bestChunkIds.length === 1 ? "" : "s" + }`, + })); +} + export async function POST(request: Request) { try { const body = searchSchema.parse(await request.json()); @@ -29,37 +85,43 @@ export async function POST(request: Request) { visualEvidence: buildVisualEvidence(results), smartPanel: buildSmartPanel(body.query, results), relatedDocuments: [], + documentMatches: body.mode === "documents" ? buildDocumentMatchesFromResults(results, body.documentLimit) : [], demoMode: true, }); } const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); const search = await searchChunksWithTelemetry({ query: body.query, - topK: body.topK ?? 8, + topK: body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8), documentId: body.documentId, documentIds: body.documentIds, - ownerId: user.id, + ownerId: undefined, }); - const results = diversifySearchResults(search.results, body.topK ?? 8); + const resultLimit = + body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8); + const results = diversifySearchResults(search.results, resultLimit, 4, true); const relatedDocuments = body.includeRelatedDocuments ? await fetchRelatedDocuments({ supabase, - ownerId: user.id, + ownerId: undefined, query: body.query, results, + limit: body.mode === "documents" ? body.documentLimit : undefined, }) : []; const smartPanel = buildSmartPanel(body.query, results); + const documentMatches = body.mode === "documents" ? relatedDocuments.map(toDocumentMatch) : []; return NextResponse.json({ results, visualEvidence: buildVisualEvidence(results), relatedDocuments, + documentMatches, smartPanel: { ...smartPanel, relatedDocuments }, telemetry: { + query_class: search.telemetry.query_class, retrieval_strategy: search.telemetry.retrieval_strategy, search_cache_hit: search.telemetry.search_cache_hit, embedding_skipped: search.telemetry.embedding_skipped, @@ -68,12 +130,20 @@ export async function POST(request: Request) { embedding_latency_ms: search.telemetry.embedding_latency_ms, supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms, rerank_latency_ms: search.telemetry.rerank_latency_ms, + memory_card_count: search.telemetry.memory_card_count, + memory_top_score: search.telemetry.memory_top_score, }, }); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof Error && error.message.trim()) { + return jsonError( + new PublicApiError("Search failed. Retry with a narrower question.", 500, { code: error.name }), + 500, + ); } - return jsonError(error, 400); + return jsonError(error, 500); } } diff --git a/src/app/api/setup-status/route.ts b/src/app/api/setup-status/route.ts index 02b855f2d..708407806 100644 --- a/src/app/api/setup-status/route.ts +++ b/src/app/api/setup-status/route.ts @@ -86,12 +86,7 @@ async function readWorkerStatus() { const label = "npm run worker running"; if (!requiredSupabaseEnvPresent) { - return check( - "worker", - label, - "unknown", - "Worker status cannot be inferred until Supabase is configured.", - ); + return check("worker", label, "unknown", "Worker status cannot be inferred until Supabase is configured."); } if (!supabaseProjectCanBeQueried) { @@ -107,7 +102,10 @@ async function readWorkerStatus() { const supabase = createAdminClient(); const [latestResult, activeResult] = await Promise.all([ supabase.from("ingestion_jobs").select("status,updated_at").order("updated_at", { ascending: false }).limit(1), - supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]), + supabase + .from("ingestion_jobs") + .select("id", { count: "exact", head: true }) + .in("status", ["pending", "processing"]), ]); if (latestResult.error || activeResult.error) { diff --git a/src/app/layout.tsx b/src/app/layout.tsx index c9e586c6d..78b847a04 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -1,6 +1,5 @@ import type { Metadata, Viewport } from "next"; import { Geist, Geist_Mono } from "next/font/google"; -import Script from "next/script"; import { AuthProvider } from "@/lib/supabase/client"; import "./globals.css"; @@ -30,16 +29,6 @@ export default function RootLayout({ }: Readonly<{ children: React.ReactNode; }>) { - const themeScript = ` - (() => { - try { - const stored = localStorage.getItem("clinical-kb-theme"); - const dark = stored === "dark" || (!stored && window.matchMedia("(prefers-color-scheme: dark)").matches); - document.documentElement.classList.toggle("dark", dark); - } catch {} - })(); - `; - return ( -