diff --git a/.env.example b/.env.example index d5e826e49..dadd4d6ae 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,17 @@ SUPABASE_PROJECT_NAME=Clinical KB Database NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=your-publishable-or-anon-key SUPABASE_SERVICE_ROLE_KEY=your-service-role-key +# Local-only no-auth mode (development only). +# Enable one or both of these to load real data without per-request browser auth: +# - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server) +# - LOCAL_NO_AUTH (server only) +#NEXT_PUBLIC_LOCAL_NO_AUTH=false +#LOCAL_NO_AUTH=false + +# Optional local no-auth owner identity. Set one of these in local dev. +#LOCAL_NO_AUTH_OWNER_ID= +#LOCAL_NO_AUTH_OWNER_EMAIL= + # OpenAI direct API. This app sends extracted guideline text and extracted images # to OpenAI for embeddings, captioning, and grounded answer generation. OPENAI_API_KEY=replace-with-openai-api-key @@ -14,7 +25,7 @@ OPENAI_EMBEDDING_MODEL=text-embedding-3-small OPENAI_ANSWER_MODEL=gpt-5.4-mini OPENAI_FAST_ANSWER_MODEL=gpt-5.4-mini OPENAI_STRONG_ANSWER_MODEL=gpt-5.4 -OPENAI_MAX_OUTPUT_TOKENS=900 +OPENAI_MAX_OUTPUT_TOKENS=1400 OPENAI_QUERY_CACHE_SIZE=200 OPENAI_VISION_MODEL=gpt-5.4-mini OPENAI_REQUEST_TIMEOUT_MS=45000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58ec3f823..efa46cf57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: 22 cache: npm diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 42a422724..367b11810 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -19,11 +19,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Scan for secrets - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 1bd3f6f39..bd2f576bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,3 +27,103 @@ This version has breaking changes — APIs, conventions, and file structure may - Older unused project ref `qjgitjyhxrwxsrydablr` belongs to `Database`; treat it as stale and do not use it. - Run `npm run check:supabase-project` after changing Supabase env values. + + + +# `upload` shortcut + +When the user types exactly: + +upload + +as the entire task message, treat it as a shortcut for the safe Git handoff workflow below. + +The goal is to leave useful completed work safely committed and, where safe, pushed to the current feature branch. The goal is not to merge into main, delete branches, discard work, force-push, close PRs, deploy, or perform destructive cleanup without explicit user confirmation. + +## Protected and base branches + +Treat `main`, `master`, `develop`, and `release/*` as protected/base branches for this workflow. + +If `upload` is run while on `main`, automatically create or use a branch named exactly `temporary` before staging, committing, or pushing, then continue the upload workflow from `temporary`: + +- If neither local `temporary` nor `origin/temporary` exists, run `git switch -c temporary`. +- If local `temporary` exists and is not checked out in another worktree, switch to it only when it is clearly safe. +- If `origin/temporary` exists, use it only when it is clearly the matching intended branch. +- If any `temporary` branch state is ambiguous, diverged, checked out elsewhere, or unsafe, stop and ask instead of overwriting. + +If already on a non-protected feature branch, continue using that branch. + +## Required inspection + +Start with read-only inspection before making changes. Check: + +- Current branch or detached HEAD state +- `git status` +- Staged, unstaged, and untracked files +- Recent commits relevant to the current branch +- Remote configuration and upstream branch +- Whether the branch is ahead, behind, or diverged +- Whether the current branch appears protected/base +- Other Git worktrees, if detectable +- Available checks such as tests, lint, type check, or build scripts +- Existing branch, commit, PR, and release-flow conventions + +Do not assume branch names, remotes, package managers, test commands, deployment targets, or project structure. Inspect first. + +## Safe actions allowed without further confirmation + +When the repository state makes it clearly safe, you may: + +- Stage coherent completed changes that clearly belong together +- Create one or more logical commits with clear messages based on the diff +- Fast-forward pull only when there are no local commits or conflict risks +- Push the current non-protected feature branch if it has a valid upstream +- Set an upstream for the current feature branch only when the correct remote and branch name are obvious +- Leave the worktree clean by committing safe completed changes + +## Actions requiring explicit confirmation + +Do not perform these without asking the user first: + +- `git reset --hard` +- `git clean -fd` or other destructive cleanup +- Discarding, overwriting, or reverting uncommitted changes +- Deleting local or remote branches +- Renaming branches +- Force-pushing +- Rebasing a shared/public branch +- Resolving divergent branch history +- Merging into `main`, `master`, `develop`, `release/*`, or any protected/base branch +- Closing pull requests +- Changing GitHub default branch, branch protection, repository settings, or deployment settings +- Modifying production data or deployment configuration +- Committing secrets, credentials, tokens, private keys, or sensitive local configuration +- Updating branch references where the correct replacement branch is ambiguous + +If any of these seem necessary, stop and report what is risky, why it is risky, the recommended next step, and the exact confirmation needed. + +## Mixed, suspicious, or unsafe changes + +Do not automatically commit files that look like `.env` files, credentials, secrets, logs, caches, build artifacts, editor or OS files, temporary/debug files, or generated files not normally committed by this project. Report only the path and concern for possible secrets; never print secret values. + +If changes appear unrelated, incomplete, experimental, or WIP, do not commit everything together automatically. Commit only clearly coherent completed changes when safe; otherwise summarize the groups and ask what should be included. + +## Branch cleanup and reference updates + +If stale, inappropriate, merged, or unnecessary branches are detected, list cleanup candidates but do not delete or rename branches automatically. + +Before recommending deletion or rename, audit accessible references including `.github/workflows/*`, CI/CD config, deployment config, scripts, package scripts, docs, release notes or release scripts, safe environment/config files, branch-specific config, open PR metadata if accessible, and GitHub branch protection/default branch metadata if safely accessible. + +Update repo-tracked references to a renamed or replacement branch only when the old branch reference is clearly found, the replacement is obvious, the change is low-risk, and the user has approved the branch rename or deletion. If the replacement is unclear, report the reference and ask what it should point to. + +## Syncing and verification + +Do not rebase, merge, or resolve remote divergence automatically. Fast-forward pulls are allowed only when clearly safe. Push only the current non-protected feature branch when clearly safe. + +Run the smallest relevant checks that are available and appropriate, such as tests, lint, type check, or build checks. Do not claim checks passed unless they were actually run. If checks cannot be run, explain why and state the command that would normally be used. + +## Final report + +After completing `upload`, summarize the current branch and worktree state, whether the worktree is clean, what changed, files committed, commit hash and message if created, whether anything was pushed, remote branch and likely PR target, checks run and results, checks not run and why, branch cleanup candidates, branch references found or updated, risky actions skipped, and exact confirmation needed for any recommended follow-up. + + diff --git a/package.json b/package.json index aefa51570..547b235f6 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "check:indexing": "tsx scripts/check-indexing.ts", "eval:rag": "tsx scripts/eval-rag.ts", "eval:search": "tsx scripts/eval-search.ts", + "cleanup:storage": "tsx scripts/cleanup-storage.ts", + "purge:query-logs": "tsx scripts/purge-query-logs.ts", + "audit:tables": "tsx scripts/audit-tables.ts", "samples": "tsx scripts/generate-sample-documents.ts", "samples:check": "tsx scripts/check-sample-extraction.ts" }, diff --git a/scripts/audit-tables.ts b/scripts/audit-tables.ts new file mode 100644 index 000000000..b6a20d883 --- /dev/null +++ b/scripts/audit-tables.ts @@ -0,0 +1,188 @@ +import { loadEnvConfig } from "@next/env"; +import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils"; +import { assessClinicalImageUse } from "@/lib/image-filtering"; + +loadEnvConfig(process.cwd()); + +type Args = { + ownerEmail?: string; + allOwners: boolean; + document?: string; + limit: number; + failOnMissed: boolean; +}; + +function parseArgs(): Args { + const args: Args = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, allOwners: !process.env.RAG_EVAL_OWNER_EMAIL, limit: 200, failOnMissed: false }; + const tokens = process.argv.slice(2); + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + const value = tokens[index + 1]; + if (token === "--owner-email") { + args.ownerEmail = value; + args.allOwners = false; + index += 1; + } else if (token === "--all-owners") { + args.allOwners = true; + } else if (token === "--document") { + args.document = value; + index += 1; + } else if (token === "--limit") { + args.limit = Math.max(1, Math.min(Number(value) || 200, 1000)); + index += 1; + } else if (token === "--fail-on-missed") { + args.failOnMissed = true; + } + } + return args; +} + +function textSuggestsTable(text: string) { + return /\b(table\s+\d+[a-z]?|appendix\s+\d+[a-z]?|roles?\s+and\s+responsibilities|score\b.*\bmanagement|observation|medication|dose|frequency)\b/i.test( + text, + ); +} + +function compactTitle(title: string) { + return title.length > 72 ? `${title.slice(0, 69).trim()}...` : title; +} + +async function main() { + const args = parseArgs(); + if (!args.ownerEmail && !args.allOwners) throw new Error('Provide --owner-email "you@example.com" or --all-owners.'); + + const supabase = await loadAdminClient(); + const ownerId = args.ownerEmail && !args.allOwners ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + + let documentQuery = supabase + .from("documents") + .select("id,title,file_name,status,page_count,image_count") + .order("created_at", { ascending: true }) + .limit(args.limit); + if (ownerId) documentQuery = documentQuery.eq("owner_id", ownerId); + if (args.document) { + documentQuery = documentQuery.or( + `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`, + ); + } + + const { data: documents, error: documentsError } = await documentQuery; + if (documentsError) throw new Error(documentsError.message); + const documentIds = (documents ?? []).map((document) => document.id as string); + if (documentIds.length === 0) { + console.log("No documents matched the audit filters."); + return; + } + + const [imagesResult, pagesResult] = await Promise.all([ + supabase + .from("document_images") + .select("document_id,source_kind,searchable,image_type,clinical_relevance_score,metadata") + .in("document_id", documentIds) + .neq("image_type", "logo_decorative"), + supabase.from("document_pages").select("document_id,page_number,text").in("document_id", documentIds), + ]); + if (imagesResult.error) throw new Error(imagesResult.error.message); + if (pagesResult.error) throw new Error(pagesResult.error.message); + + const counts = new Map(); + for (const documentId of documentIds) { + counts.set(documentId, { tables: 0, clinicalTables: 0, adminTables: 0, searchableAdminTables: 0, images: 0 }); + } + for (const image of imagesResult.data ?? []) { + const documentId = String(image.document_id); + const current = counts.get(documentId) ?? { + tables: 0, + clinicalTables: 0, + adminTables: 0, + searchableAdminTables: 0, + images: 0, + }; + const metadata = image.metadata && typeof image.metadata === "object" ? (image.metadata as Record) : {}; + const useClass = String( + metadata.clinical_use_class ?? + assessClinicalImageUse({ + imageType: image.image_type, + searchable: image.searchable, + clinicalRelevanceScore: image.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: typeof metadata.table_role === "string" ? metadata.table_role : null, + tableText: + typeof metadata.table_text === "string" + ? metadata.table_text + : typeof metadata.table_text_snippet === "string" + ? metadata.table_text_snippet + : null, + }).clinical_use_class, + ); + if (image.searchable !== false) current.images += 1; + if (image.source_kind === "table_crop") { + current.tables += 1; + if (useClass === "clinical_evidence" && image.searchable !== false) current.clinicalTables += 1; + if (["administrative", "reference"].includes(useClass)) current.adminTables += 1; + if (["administrative", "reference"].includes(useClass) && image.searchable !== false) { + current.searchableAdminTables += 1; + } + } + counts.set(documentId, current); + } + + const tableMarkerPages = new Map>(); + for (const page of pagesResult.data ?? []) { + if (!textSuggestsTable(String(page.text ?? ""))) continue; + const documentId = String(page.document_id); + const pages = tableMarkerPages.get(documentId) ?? new Set(); + const pageNumber = Number(page.page_number); + if (Number.isFinite(pageNumber)) pages.add(pageNumber); + tableMarkerPages.set(documentId, pages); + } + + const possibleMisses: string[] = []; + const searchableAdminIssues: string[] = []; + console.log(`Table audit for ${documents?.length ?? 0} documents`); + for (const document of documents ?? []) { + const documentCounts = counts.get(document.id) ?? { + tables: 0, + clinicalTables: 0, + adminTables: 0, + searchableAdminTables: 0, + images: 0, + }; + const markerPages = Array.from(tableMarkerPages.get(document.id) ?? []).sort((a, b) => a - b); + if (markerPages.length > 0 && documentCounts.tables === 0) { + possibleMisses.push(`${document.file_name}: table-like text on pages ${markerPages.slice(0, 8).join(", ")}`); + } + if (documentCounts.searchableAdminTables > 0) { + searchableAdminIssues.push(`${document.file_name}: searchable admin/reference tables=${documentCounts.searchableAdminTables}`); + } + console.log( + [ + compactTitle(document.file_name ?? document.title ?? document.id), + `status=${document.status}`, + `pages=${document.page_count ?? 0}`, + `tables=${documentCounts.tables}`, + `clinicalTables=${documentCounts.clinicalTables}`, + `adminReferenceTables=${documentCounts.adminTables}`, + `searchableAdminReference=${documentCounts.searchableAdminTables}`, + `searchableImages=${documentCounts.images}`, + markerPages.length ? `tableTextPages=${markerPages.slice(0, 8).join(",")}` : "tableTextPages=none", + ].join(" | "), + ); + } + + console.log(`Possible missed table documents: ${possibleMisses.length}`); + for (const item of possibleMisses.slice(0, 20)) console.log(`- ${item}`); + console.log(`Searchable admin/reference table issues: ${searchableAdminIssues.length}`); + for (const item of searchableAdminIssues.slice(0, 20)) console.log(`- ${item}`); + if (searchableAdminIssues.length > 0) { + throw new Error("Table audit found admin/reference tables still marked searchable."); + } + if (args.failOnMissed && possibleMisses.length > 0) { + throw new Error("Table audit found documents with table-like text but no retained table crops."); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index 7efdeb6ee..cdd7c2d4c 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -2,14 +2,108 @@ import { loadEnvConfig } from "@next/env"; loadEnvConfig(process.cwd()); -async function main() { - const [{ env, requireOpenAIEnv, requireServerEnv }, { createAdminClient }, { checkPythonPdfPrerequisites }] = - await Promise.all([ - import("@/lib/env"), - import("@/lib/supabase/admin"), - import("../worker/prerequisites"), +type MetadataRow = { document_id: string; metadata?: unknown; source?: string | null }; +type QueryResponse = { + data: T[] | null; + error: { message: string } | null; + count?: number | null; +}; +type QueryBuilder = PromiseLike> & { + eq(column: string, value: unknown): QueryBuilder; + gt(column: string, value: unknown): QueryBuilder; + in(column: string, values: unknown[]): QueryBuilder; + is(column: string, value: unknown): QueryBuilder; + order(column: string, options?: Record): QueryBuilder; + range(from: number, to: number): PromiseLike>; + limit(count: number): PromiseLike>; +}; +type SupabaseLike = { + from(table: string): { + select(columns: string, options?: Record): QueryBuilder; + }; +}; + +function metadataRecord(metadata: unknown): Record { + return metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? (metadata as Record) + : {}; +} + +function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) { + return metadataRecord(metadata).rag_enrichment_version === expectedVersion; +} + +function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) { + const record = metadataRecord(metadata); + return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion; +} + +function strictEnrichmentVersionRequired() { + return process.argv.includes("--strict-enrichment-version") || process.env.RAG_REQUIRE_CURRENT_ENRICHMENT_VERSION === "1"; +} + +async function loadEnrichmentRows(supabase: SupabaseLike, documentIds: string[]) { + const summaries: MetadataRow[] = []; + const labels: MetadataRow[] = []; + + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + const [summaryResult, labelResult] = await Promise.all([ + supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids), + supabase.from("document_labels").select("document_id,source,metadata").in("document_id", ids), ]); + if (summaryResult.error) throw new Error(summaryResult.error.message); + if (labelResult.error) throw new Error(labelResult.error.message); + summaries.push(...((summaryResult.data ?? []) as MetadataRow[])); + labels.push(...((labelResult.data ?? []) as MetadataRow[])); + } + + return { summaries, labels }; +} + +async function loadRowsForDocuments(supabase: SupabaseLike, table: string, select: string, documentIds: string[]) { + const rows: MetadataRow[] = []; + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + for (let rangeStart = 0; ; rangeStart += 1000) { + const { data, error } = await supabase + .from(table) + .select(select) + .in("document_id", ids) + .range(rangeStart, rangeStart + 999); + if (error) throw new Error(error.message); + rows.push(...((data ?? []) as MetadataRow[])); + if (!data || data.length < 1000) break; + } + } + return rows; +} + +async function loadDeepMemoryRows(supabase: SupabaseLike, documentIds: string[]) { + const [sections, memoryCards, chunks] = await Promise.all([ + loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_chunks", "document_id,metadata", documentIds), + ]); + return { sections, memoryCards, chunks }; +} + +async function main() { + const [ + { env, requireOpenAIEnv, requireServerEnv }, + { createAdminClient }, + { checkPythonPdfPrerequisites }, + { ragEnrichmentVersion }, + { ragDeepMemoryVersion }, + ] = await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("../worker/prerequisites"), + import("@/lib/document-enrichment"), + import("@/lib/deep-memory"), + ]); + requireServerEnv(); requireOpenAIEnv(); @@ -23,11 +117,188 @@ async function main() { if (batchError) throw new Error(batchError.message); const { error: jobError } = await supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts").limit(1); if (jobError) throw new Error(jobError.message); + const { error: cleanupTableError } = await supabase.from("storage_cleanup_jobs").select("id,status").limit(1); + if (cleanupTableError) throw new Error(cleanupTableError.message); + + const { data: documents, error: documentsError } = await supabase + .from("documents") + .select("id,owner_id,title,content_hash,status,page_count,chunk_count,metadata"); + if (documentsError) throw new Error(documentsError.message); + + const supabaseForChecks = supabase as unknown as SupabaseLike; + const indexedDocuments = (documents ?? []).filter((document) => document.status === "indexed"); + const enrichmentRows = await loadEnrichmentRows( + supabaseForChecks, + indexedDocuments.map((document) => document.id), + ); + const deepMemoryRows = await loadDeepMemoryRows( + supabaseForChecks, + indexedDocuments.map((document) => document.id), + ); + const summariesByDocument = new Map(enrichmentRows.summaries.map((row) => [row.document_id, row])); + const labelRowsByDocument = new Map(); + const sectionRowsByDocument = new Map(); + const memoryRowsByDocument = new Map(); + const chunkRowsByDocument = new Map(); + for (const label of enrichmentRows.labels) { + labelRowsByDocument.set(label.document_id, [...(labelRowsByDocument.get(label.document_id) ?? []), label]); + } + for (const section of deepMemoryRows.sections) { + sectionRowsByDocument.set(section.document_id, [...(sectionRowsByDocument.get(section.document_id) ?? []), section]); + } + for (const card of deepMemoryRows.memoryCards) { + memoryRowsByDocument.set(card.document_id, [...(memoryRowsByDocument.get(card.document_id) ?? []), card]); + } + for (const chunk of deepMemoryRows.chunks) { + chunkRowsByDocument.set(chunk.document_id, [...(chunkRowsByDocument.get(chunk.document_id) ?? []), chunk]); + } + + const duplicateHashGroups = new Map(); + for (const document of documents ?? []) { + if (!document.owner_id || !document.content_hash) continue; + const key = `${document.owner_id}:${document.content_hash}`; + duplicateHashGroups.set(key, [...(duplicateHashGroups.get(key) ?? []), document.title ?? document.id]); + } + const duplicateGroups = Array.from(duplicateHashGroups.values()).filter((titles) => titles.length > 1); + const emptyIndexedDocuments = (documents ?? []).filter( + (document) => + document.status === "indexed" && ((document.page_count ?? 0) === 0 || (document.chunk_count ?? 0) === 0), + ); + const documentsWithChunkCountMismatch = indexedDocuments.filter( + (document) => (chunkRowsByDocument.get(document.id) ?? []).length !== (document.chunk_count ?? 0), + ); + const documentsMissingSummaries = indexedDocuments.filter((document) => !summariesByDocument.has(document.id)); + const documentsMissingGeneratedLabels = indexedDocuments.filter((document) => + (labelRowsByDocument.get(document.id) ?? []).every((label) => label.source !== "generated"), + ); + const documentsWithCurrentEnrichmentVersion = indexedDocuments.filter((document) => { + const summary = summariesByDocument.get(document.id); + const generatedLabels = (labelRowsByDocument.get(document.id) ?? []).filter((label) => label.source === "generated"); + return ( + hasCurrentEnrichmentVersion(document.metadata, ragEnrichmentVersion) && + hasCurrentEnrichmentVersion(summary?.metadata, ragEnrichmentVersion) && + generatedLabels.some((label) => hasCurrentEnrichmentVersion(label.metadata, ragEnrichmentVersion)) + ); + }); + const documentsMissingCurrentEnrichmentVersion = + indexedDocuments.length - documentsWithCurrentEnrichmentVersion.length; + const documentsMissingSections = indexedDocuments.filter( + (document) => (sectionRowsByDocument.get(document.id) ?? []).length === 0, + ); + const documentsMissingMemoryCards = indexedDocuments.filter( + (document) => (memoryRowsByDocument.get(document.id) ?? []).length === 0, + ); + const documentsWithCurrentDeepMemoryVersion = indexedDocuments.filter((document) => { + const sections = sectionRowsByDocument.get(document.id) ?? []; + const memoryCards = memoryRowsByDocument.get(document.id) ?? []; + const chunks = chunkRowsByDocument.get(document.id) ?? []; + return ( + hasCurrentMemoryVersion(document.metadata, ragDeepMemoryVersion) && + sections.some((section) => hasCurrentMemoryVersion(section.metadata, ragDeepMemoryVersion)) && + memoryCards.some((card) => hasCurrentMemoryVersion(card.metadata, ragDeepMemoryVersion)) && + chunks.length > 0 && + chunks.every((chunk) => hasCurrentMemoryVersion(chunk.metadata, ragDeepMemoryVersion)) + ); + }); + const documentsMissingCurrentDeepMemoryVersion = + indexedDocuments.length - documentsWithCurrentDeepMemoryVersion.length; + const requireCurrentEnrichmentVersion = strictEnrichmentVersionRequired(); + + const [ + missingEmbeddingResult, + failedJobsResult, + activeJobsResult, + imageCountResult, + searchableImageResult, + oversizedBatchesResult, + cleanupIssuesResult, + ] = await Promise.all([ + supabase.from("document_chunks").select("id", { count: "exact", head: true }).is("embedding", null), + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).eq("status", "failed"), + supabase + .from("ingestion_jobs") + .select("id", { count: "exact", head: true }) + .in("status", ["pending", "processing"]), + supabase.from("document_images").select("id", { count: "exact", head: true }), + supabase.from("document_images").select("id", { count: "exact", head: true }).eq("searchable", true), + supabase + .from("import_batches") + .select("id,name,total_files,status") + .gt("total_files", 150) + .in("status", ["queued", "processing"]), + supabase + .from("storage_cleanup_jobs") + .select("id,status,last_error") + .in("status", ["pending", "failed"]) + .order("created_at", { ascending: true }) + .limit(5), + ]); + + for (const result of [ + missingEmbeddingResult, + failedJobsResult, + activeJobsResult, + imageCountResult, + searchableImageResult, + oversizedBatchesResult, + cleanupIssuesResult, + ]) { + if (result.error) throw new Error(result.error.message); + } + + const issues: string[] = []; + if (duplicateGroups.length > 0) issues.push(`duplicate content-hash groups: ${duplicateGroups.length}`); + if ((missingEmbeddingResult.count ?? 0) > 0) + issues.push(`chunks missing embeddings: ${missingEmbeddingResult.count}`); + if (emptyIndexedDocuments.length > 0) issues.push(`empty indexed documents: ${emptyIndexedDocuments.length}`); + if (documentsWithChunkCountMismatch.length > 0) + issues.push(`indexed document chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`); + if (documentsMissingSummaries.length > 0) + issues.push(`indexed documents missing summaries: ${documentsMissingSummaries.length}`); + if (documentsMissingGeneratedLabels.length > 0) + issues.push(`indexed documents missing generated labels: ${documentsMissingGeneratedLabels.length}`); + if (documentsMissingSections.length > 0) issues.push(`indexed documents missing sections: ${documentsMissingSections.length}`); + if (documentsMissingMemoryCards.length > 0) + issues.push(`indexed documents missing memory cards: ${documentsMissingMemoryCards.length}`); + if (requireCurrentEnrichmentVersion && documentsMissingCurrentEnrichmentVersion > 0) { + issues.push(`indexed documents missing current enrichment version: ${documentsMissingCurrentEnrichmentVersion}`); + } + if (requireCurrentEnrichmentVersion && documentsMissingCurrentDeepMemoryVersion > 0) { + issues.push(`indexed documents missing current deep-memory version: ${documentsMissingCurrentDeepMemoryVersion}`); + } + if ((failedJobsResult.count ?? 0) > 0) issues.push(`failed ingestion jobs: ${failedJobsResult.count}`); + if ((oversizedBatchesResult.data ?? []).length > 0) { + issues.push(`active oversized import batches: ${(oversizedBatchesResult.data ?? []).length}`); + } + if ((cleanupIssuesResult.data ?? []).length > 0) { + issues.push(`pending or failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); + } console.log("Indexing prerequisites ready."); console.log("Supabase bulk ingestion tables are reachable."); console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`); console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`); + console.log(`Documents: ${(documents ?? []).length}; empty indexed: ${emptyIndexedDocuments.length}`); + console.log(`Chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`); + console.log( + `Enrichment coverage: summaries missing=${documentsMissingSummaries.length}; generated labels missing=${documentsMissingGeneratedLabels.length}`, + ); + console.log( + `RAG enrichment version: ${documentsWithCurrentEnrichmentVersion.length}/${indexedDocuments.length} indexed current (${ragEnrichmentVersion}); strict=${requireCurrentEnrichmentVersion ? "yes" : "no"}`, + ); + console.log( + `Deep memory version: ${documentsWithCurrentDeepMemoryVersion.length}/${indexedDocuments.length} indexed current (${ragDeepMemoryVersion}); sections missing=${documentsMissingSections.length}; memory cards missing=${documentsMissingMemoryCards.length}`, + ); + console.log(`Duplicate content-hash groups: ${duplicateGroups.length}`); + console.log(`Chunks missing embeddings: ${missingEmbeddingResult.count ?? 0}`); + console.log(`Failed jobs: ${failedJobsResult.count ?? 0}; pending/processing jobs: ${activeJobsResult.count ?? 0}`); + console.log(`Images: ${imageCountResult.count ?? 0}; searchable: ${searchableImageResult.count ?? 0}`); + console.log(`Active oversized batches: ${(oversizedBatchesResult.data ?? []).length}`); + console.log(`Pending/failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); + + if (issues.length > 0) { + throw new Error(`Indexing readiness check failed: ${issues.join("; ")}`); + } } main().catch((error) => { diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts new file mode 100644 index 000000000..27389bc13 --- /dev/null +++ b/scripts/cleanup-storage.ts @@ -0,0 +1,120 @@ +import { loadEnvConfig } from "@next/env"; +import { loadAdminClient } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +type CleanupArgs = { + limit: number; + dryRun: boolean; +}; + +type CleanupJob = { + id: string; + document_bucket: string | null; + document_paths: string[] | null; + image_bucket: string | null; + image_paths: string[] | null; + attempts: number; +}; + +function parseArgs(argv: string[]): CleanupArgs { + const args: CleanupArgs = { limit: 50, dryRun: false }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--dry-run") { + args.dryRun = true; + continue; + } + if (token === "--limit") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("Missing value for --limit"); + args.limit = Number.parseInt(value, 10); + index += 1; + } + } + + if (!Number.isInteger(args.limit) || args.limit <= 0) throw new Error("--limit must be a positive integer."); + return args; +} + +async function removePaths(args: { + supabase: Awaited>; + bucket: string; + paths: string[]; +}) { + let removed = 0; + const warnings: string[] = []; + const uniquePaths = Array.from(new Set(args.paths.filter(Boolean))); + + for (let start = 0; start < uniquePaths.length; start += 1000) { + const batch = uniquePaths.slice(start, start + 1000); + const { data, error } = await args.supabase.storage.from(args.bucket).remove(batch); + if (error) { + warnings.push(`${args.bucket}: ${error.message}`); + } else { + removed += data?.length ?? 0; + } + } + + return { removed, warnings }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const supabase = await loadAdminClient(); + const { data, error } = await supabase + .from("storage_cleanup_jobs") + .select("id,document_bucket,document_paths,image_bucket,image_paths,attempts") + .in("status", ["pending", "failed"]) + .order("created_at", { ascending: true }) + .limit(args.limit); + + if (error) throw new Error(error.message); + const jobs = (data ?? []) as CleanupJob[]; + console.log(`Found ${jobs.length} storage cleanup job(s).`); + if (args.dryRun || jobs.length === 0) return; + + let completed = 0; + let failed = 0; + + for (const job of jobs) { + const documentCleanup = await removePaths({ + supabase, + bucket: job.document_bucket ?? "clinical-documents", + paths: job.document_paths ?? [], + }); + const imageCleanup = await removePaths({ + supabase, + bucket: job.image_bucket ?? "clinical-images", + paths: job.image_paths ?? [], + }); + const warnings = [...documentCleanup.warnings, ...imageCleanup.warnings]; + const nextStatus = warnings.length > 0 ? "failed" : "completed"; + const { error: updateError } = await supabase + .from("storage_cleanup_jobs") + .update({ + status: nextStatus, + attempts: job.attempts + 1, + storage_removed: documentCleanup.removed + imageCleanup.removed, + last_error: warnings.length ? warnings.join("; ") : null, + completed_at: nextStatus === "completed" ? new Date().toISOString() : null, + metadata: { + operation: "storage_cleanup_retry", + storage_warnings: warnings, + }, + }) + .eq("id", job.id); + + if (updateError) throw new Error(updateError.message); + if (nextStatus === "completed") completed += 1; + else failed += 1; + } + + console.log(`Storage cleanup complete: ${completed} completed, ${failed} failed.`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index 35cbd065e..936e59ca8 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -5,12 +5,16 @@ loadEnvConfig(process.cwd()); type EnrichArgs = { ownerEmail?: string; + ownerId?: string; + allOwners: boolean; mode: string; limit: number; documentId?: string; + includeCurrent: boolean; }; type SupabaseAdmin = Awaited>; +type MetadataRow = { id?: string; document_id: string; metadata?: unknown; source?: string | null }; async function loadAdminClient() { const { createAdminClient } = await import("@/lib/supabase/admin"); @@ -20,18 +24,30 @@ async function loadAdminClient() { function parseArgs(argv: string[]): EnrichArgs { const args: EnrichArgs = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, + allOwners: !process.env.RAG_EVAL_OWNER_EMAIL && !process.env.RAG_EVAL_OWNER_ID && !process.env.LOCAL_NO_AUTH_OWNER_ID, mode: "summaries-labels-images", limit: 25, + includeCurrent: false, }; for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; if (!token.startsWith("--")) continue; + if (token === "--all-owners") { + args.allOwners = true; + continue; + } + if (token === "--include-current") { + args.includeCurrent = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); index += 1; if (token === "--owner-email") args.ownerEmail = value; + if (token === "--owner-id") args.ownerId = value; if (token === "--mode") args.mode = value; if (token === "--limit") args.limit = Number.parseInt(value, 10); if (token === "--document-id") args.documentId = value; @@ -56,15 +72,15 @@ async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) { throw new Error(`No Supabase Auth user found for ${email}. Sign in once before enriching documents.`); } -async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId: string) { +async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId?: string) { let query = supabase .from("documents") .select("id,owner_id,title,file_name,source_path,status,metadata") - .eq("owner_id", ownerId) .eq("status", "indexed") .order("created_at", { ascending: true }) - .limit(args.limit); + .limit(args.documentId ? 1 : Math.max(args.limit * 10, 1000)); + if (ownerId) query = query.eq("owner_id", ownerId); if (args.documentId) query = query.eq("id", args.documentId); const { data, error } = await query; @@ -72,39 +88,174 @@ async function loadDocuments(supabase: SupabaseAdmin, args: EnrichArgs, ownerId: return data ?? []; } +async function loadEnrichmentCoverage(supabase: SupabaseAdmin, documentIds: string[]) { + const summaries: MetadataRow[] = []; + const labels: MetadataRow[] = []; + + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + const [summaryResult, labelResult] = await Promise.all([ + supabase.from("document_summaries").select("document_id,metadata").in("document_id", ids), + supabase.from("document_labels").select("id,document_id,source,metadata").in("document_id", ids), + ]); + + if (summaryResult.error) throw new Error(summaryResult.error.message); + if (labelResult.error) throw new Error(labelResult.error.message); + summaries.push(...((summaryResult.data ?? []) as MetadataRow[])); + labels.push(...((labelResult.data ?? []) as MetadataRow[])); + } + + const coverage = new Map(); + for (const documentId of documentIds) coverage.set(documentId, { labels: [] }); + for (const summary of summaries) { + coverage.set(summary.document_id, { ...(coverage.get(summary.document_id) ?? { labels: [] }), summary }); + } + for (const label of labels) { + const existing = coverage.get(label.document_id) ?? { labels: [] }; + coverage.set(label.document_id, { ...existing, labels: [...existing.labels, label] }); + } + + return coverage; +} + +async function loadRowsForDocuments(supabase: SupabaseAdmin, table: string, select: string, documentIds: string[]) { + const rows: MetadataRow[] = []; + for (let start = 0; start < documentIds.length; start += 100) { + const ids = documentIds.slice(start, start + 100); + for (let rangeStart = 0; ; rangeStart += 1000) { + const { data, error } = await supabase + .from(table) + .select(select) + .in("document_id", ids) + .range(rangeStart, rangeStart + 999); + if (error) throw new Error(error.message); + rows.push(...((data ?? []) as unknown as MetadataRow[])); + if (!data || data.length < 1000) break; + } + } + return rows; +} + +async function loadDeepMemoryCoverage(supabase: SupabaseAdmin, documentIds: string[]) { + const [sections, memoryCards] = await Promise.all([ + loadRowsForDocuments(supabase, "document_sections", "document_id,metadata", documentIds), + loadRowsForDocuments(supabase, "document_memory_cards", "document_id,metadata", documentIds), + ]); + + const coverage = new Map(); + for (const documentId of documentIds) coverage.set(documentId, { sections: [], memoryCards: [] }); + for (const section of sections) { + const existing = coverage.get(section.document_id) ?? { sections: [], memoryCards: [] }; + coverage.set(section.document_id, { ...existing, sections: [...existing.sections, section] }); + } + for (const card of memoryCards) { + const existing = coverage.get(card.document_id) ?? { sections: [], memoryCards: [] }; + coverage.set(card.document_id, { ...existing, memoryCards: [...existing.memoryCards, card] }); + } + return coverage; +} + async function loadEvidence(supabase: SupabaseAdmin, documentId: string) { - const [chunksResult, imagesResult] = await Promise.all([ - supabase + const chunks = []; + const images = []; + + for (let start = 0; ; start += 1000) { + const { data, error } = await supabase .from("document_chunks") - .select("id,page_number,chunk_index,section_heading,content") + .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata") .eq("document_id", documentId) .order("chunk_index", { ascending: true }) - .limit(24), - supabase + .range(start, start + 999); + if (error) throw new Error(error.message); + chunks.push(...(data ?? [])); + if (!data || data.length < 1000) break; + } + + for (let start = 0; ; start += 1000) { + const { data, error } = await supabase .from("document_images") - .select("id,page_number,caption,image_type,labels") + .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata") .eq("document_id", documentId) .eq("searchable", true) .order("clinical_relevance_score", { ascending: false }) - .limit(12), - ]); + .range(start, start + 999); + if (error) throw new Error(error.message); + images.push(...(data ?? [])); + if (!data || data.length < 1000) break; + } - if (chunksResult.error) throw new Error(chunksResult.error.message); - if (imagesResult.error) throw new Error(imagesResult.error.message); - return { chunks: chunksResult.data ?? [], images: imagesResult.data ?? [] }; + return { chunks, images }; } function hashBytes(bytes: Buffer) { return createHash("sha256").update(bytes).digest("hex"); } +function metadataString(metadata: unknown, key: string) { + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return null; + const value = (metadata as Record)[key]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function metadataRecord(metadata: unknown): Record { + return metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? { ...(metadata as Record) } + : {}; +} + +function hasCurrentEnrichmentVersion(metadata: unknown, expectedVersion: string) { + return metadataRecord(metadata).rag_enrichment_version === expectedVersion; +} + +function hasCurrentMemoryVersion(metadata: unknown, expectedVersion: string) { + const record = metadataRecord(metadata); + return record.rag_memory_version === expectedVersion || record.rag_indexing_version === expectedVersion; +} + +function needsEnrichmentBackfill(args: { + document: { metadata?: unknown }; + coverage?: { summary?: MetadataRow; labels: MetadataRow[] }; + memoryCoverage?: { sections: MetadataRow[]; memoryCards: MetadataRow[] }; + ragEnrichmentVersion: string; + ragDeepMemoryVersion: string; +}) { + const generatedLabels = args.coverage?.labels.filter((label) => label.source === "generated") ?? []; + const sections = args.memoryCoverage?.sections ?? []; + const memoryCards = args.memoryCoverage?.memoryCards ?? []; + const summaryMetadata = metadataRecord(args.coverage?.summary?.metadata); + const documentMetadata = metadataRecord(args.document.metadata); + return ( + !args.coverage?.summary || + generatedLabels.length === 0 || + !summaryMetadata.coverage_profile || + !documentMetadata.coverage_profile || + !hasCurrentEnrichmentVersion(args.document.metadata, args.ragEnrichmentVersion) || + !hasCurrentEnrichmentVersion(args.coverage.summary?.metadata, args.ragEnrichmentVersion) || + generatedLabels.every((label) => !hasCurrentEnrichmentVersion(label.metadata, args.ragEnrichmentVersion)) || + sections.length === 0 || + memoryCards.length === 0 || + !hasCurrentMemoryVersion(args.document.metadata, args.ragDeepMemoryVersion) || + sections.every((section) => !hasCurrentMemoryVersion(section.metadata, args.ragDeepMemoryVersion)) || + memoryCards.every((card) => !hasCurrentMemoryVersion(card.metadata, args.ragDeepMemoryVersion)) + ); +} + async function classifyExistingImages(supabase: SupabaseAdmin, documentId: string) { - const [{ env }, { cheapImageSkipReason, classifiedImageSkipReason, lightweightPerceptualHash }, { classifyAndCaptionImageFromBase64 }] = - await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]); + const [ + { env }, + { + assessClinicalImageUse, + cheapImageSkipReason, + classifiedImageSkipReason, + clinicalImagePolicyVersion, + lightweightPerceptualHash, + }, + { classifyAndCaptionImageFromBase64 }, + ] = await Promise.all([import("@/lib/env"), import("@/lib/image-filtering"), import("@/lib/openai")]); const { data: images, error } = await supabase .from("document_images") .select( - "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason", + "id,page_number,storage_path,mime_type,caption,bbox,width,height,source_kind,image_hash,image_type,searchable,clinical_relevance_score,skip_reason,labels,metadata", ) .eq("document_id", documentId) .order("page_number", { ascending: true }); @@ -116,11 +267,8 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin let skipped = 0; for (const image of images ?? []) { - if ( - (image.image_type && image.image_type !== "unclear") || - image.clinical_relevance_score > 0 || - image.skip_reason - ) { + const existingMetadata = metadataRecord(image.metadata); + if (existingMetadata.image_policy_version === clinicalImagePolicyVersion) { if (image.searchable) searchable += 1; else skipped += 1; continue; @@ -147,7 +295,13 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin bbox: image.bbox as [number, number, number, number] | null, width: image.width, height: image.height, - sourceKind: image.source_kind as "embedded" | "diagram_crop" | "page_region" | "fallback" | undefined, + sourceKind: image.source_kind as + | "embedded" + | "table_crop" + | "diagram_crop" + | "page_region" + | "fallback" + | undefined, }, }); @@ -167,70 +321,238 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin } seenHashes.add(imageHash); - const classification = await classifyAndCaptionImageFromBase64({ - base64: bytes.toString("base64"), - mimeType: image.mime_type, - nearbyText: image.caption ?? undefined, + const baseAssessment = assessClinicalImageUse({ + imageType: image.image_type, + searchable: image.searchable, + clinicalRelevanceScore: image.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + tableTitle: metadataString(image.metadata, "table_title"), + tableLabel: metadataString(image.metadata, "table_label"), + caption: image.caption, + labels: Array.isArray(image.labels) ? image.labels : [], + skipReason: image.skip_reason, + }); + const classification = + image.source_kind === "table_crop" && ["administrative", "reference"].includes(baseAssessment.clinical_use_class) + ? { + image_type: image.image_type || "clinical_table", + searchable: false, + clinical_relevance_score: 0, + labels: [], + caption: + baseAssessment.clinical_use_class === "administrative" + ? "Administrative document-control table retained for audit, not clinical evidence." + : "Reference table retained for audit, not clinical evidence.", + skip_reason: baseAssessment.clinical_use_reason, + clinical_use_class: baseAssessment.clinical_use_class, + clinical_use_reason: baseAssessment.clinical_use_reason, + clinical_signal_score: baseAssessment.clinical_signal_score, + admin_signal_score: baseAssessment.admin_signal_score, + } + : await classifyAndCaptionImageFromBase64({ + base64: bytes.toString("base64"), + mimeType: image.mime_type, + nearbyText: image.caption ?? undefined, + sourceKind: image.source_kind, + candidateType: metadataString(image.metadata, "candidate_type"), + tableLabel: metadataString(image.metadata, "table_label"), + tableTitle: metadataString(image.metadata, "table_title"), + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + }); + const finalAssessment = assessClinicalImageUse({ + imageType: classification.image_type, + searchable: classification.searchable, + clinicalRelevanceScore: classification.clinical_relevance_score, + sourceKind: image.source_kind, + tableRole: metadataString(image.metadata, "table_role"), + tableText: metadataString(image.metadata, "table_text"), + tableTitle: metadataString(image.metadata, "table_title"), + tableLabel: metadataString(image.metadata, "table_label"), + caption: classification.caption, + labels: classification.labels, + skipReason: classification.skip_reason, }); - const classifiedSkip = classifiedImageSkipReason(classification); + const classifiedSkip = classifiedImageSkipReason({ + ...classification, + searchable: finalAssessment.searchable, + clinical_relevance_score: finalAssessment.clinical_relevance_score, + clinical_use_class: finalAssessment.clinical_use_class, + clinical_use_reason: finalAssessment.clinical_use_reason, + clinical_signal_score: finalAssessment.clinical_signal_score, + admin_signal_score: finalAssessment.admin_signal_score, + }); + const retainAsAuditTable = + image.source_kind === "table_crop" && + ["administrative", "reference"].includes(finalAssessment.clinical_use_class) && + classification.image_type !== "logo_decorative"; + const nextSearchable = finalAssessment.searchable; await supabase .from("document_images") .update({ caption: classification.caption || image.caption, image_type: classification.image_type, - searchable: !classifiedSkip, - clinical_relevance_score: classifiedSkip ? 0 : classification.clinical_relevance_score, - skip_reason: classifiedSkip, + searchable: nextSearchable, + clinical_relevance_score: nextSearchable ? finalAssessment.clinical_relevance_score : 0, + skip_reason: nextSearchable ? null : classifiedSkip, image_hash: imageHash, perceptual_hash: perceptualHash, labels: classification.labels, + metadata: { + ...metadataRecord(image.metadata), + clinical_use_class: finalAssessment.clinical_use_class, + clinical_use_reason: finalAssessment.clinical_use_reason, + clinical_signal_score: finalAssessment.clinical_signal_score, + admin_signal_score: finalAssessment.admin_signal_score, + image_policy_version: clinicalImagePolicyVersion, + retained_for_audit: retainAsAuditTable || undefined, + retained_for_document_view: retainAsAuditTable || undefined, + }, }) .eq("id", image.id); - if (classifiedSkip) skipped += 1; + if (!nextSearchable) skipped += 1; else searchable += 1; } return { searchable, skipped, total: images?.length ?? 0 }; } +async function stampExistingEnrichmentVersion(args: { + supabase: SupabaseAdmin; + document: { id: string; metadata?: unknown }; + coverage: { summary?: MetadataRow; labels: MetadataRow[] }; + ragEnrichmentVersion: string; +}) { + const stampedAt = new Date().toISOString(); + const marker = { + rag_enrichment_version: args.ragEnrichmentVersion, + rag_enrichment_updated_at: stampedAt, + version_stamped_at: stampedAt, + }; + + const { error: documentError } = await args.supabase + .from("documents") + .update({ metadata: { ...metadataRecord(args.document.metadata), ...marker } }) + .eq("id", args.document.id); + if (documentError) throw new Error(documentError.message); + + if (args.coverage.summary) { + const { error: summaryError } = await args.supabase + .from("document_summaries") + .update({ metadata: { ...metadataRecord(args.coverage.summary.metadata), ...marker } }) + .eq("document_id", args.document.id); + if (summaryError) throw new Error(summaryError.message); + } + + for (const label of args.coverage.labels.filter((item) => item.source === "generated")) { + if (!label.id) continue; + const { error: labelError } = await args.supabase + .from("document_labels") + .update({ metadata: { ...metadataRecord(label.metadata), ...marker } }) + .eq("id", label.id); + if (labelError) throw new Error(labelError.message); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) { - throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); + if (!args.ownerId && !args.ownerEmail && !args.allOwners) { + throw new Error( + 'Provide --owner-id, set LOCAL_NO_AUTH_OWNER_ID or RAG_EVAL_OWNER_ID, provide --owner-email "you@example.com", or pass --all-owners.', + ); } - if (args.mode !== "summaries-labels-images") { - throw new Error("--mode currently supports summaries-labels-images."); + if (!["summaries-labels-images", "metadata-stamp", "deep-memory"].includes(args.mode)) { + throw new Error("--mode supports summaries-labels-images, deep-memory, or metadata-stamp."); } - const [{ requireOpenAIEnv, requireServerEnv }, { upsertDocumentEnrichment }, supabase] = await Promise.all([ + const [ + { requireOpenAIEnv, requireServerEnv }, + { ragEnrichmentVersion, upsertDocumentEnrichment }, + { ragDeepMemoryVersion, upsertDocumentDeepMemory }, + supabase, + ] = await Promise.all([ import("@/lib/env"), import("@/lib/document-enrichment"), + import("@/lib/deep-memory"), loadAdminClient(), ]); requireServerEnv(); - requireOpenAIEnv(); + if (args.mode === "summaries-labels-images" || args.mode === "deep-memory") requireOpenAIEnv(); + + const ownerId = + !args.allOwners && args.ownerId + ? args.ownerId + : args.ownerEmail && !args.allOwners + ? await findOwnerIdByEmail(supabase, args.ownerEmail) + : undefined; + const loadedDocuments = await loadDocuments(supabase, args, ownerId); + const coverage = await loadEnrichmentCoverage( + supabase, + loadedDocuments.map((document) => document.id), + ); + const memoryCoverage = await loadDeepMemoryCoverage( + supabase, + loadedDocuments.map((document) => document.id), + ); + const documents = loadedDocuments + .filter((document) => + args.includeCurrent + ? true + : needsEnrichmentBackfill({ + document, + coverage: coverage.get(document.id), + memoryCoverage: memoryCoverage.get(document.id), + ragEnrichmentVersion, + ragDeepMemoryVersion, + }), + ) + .slice(0, args.limit); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); - const documents = await loadDocuments(supabase, args, ownerId); - console.log(`Enriching ${documents.length} indexed document(s).`); + console.log( + `Enriching ${documents.length} indexed document(s). scope=${ownerId ? "owner" : "all"} version=${ragEnrichmentVersion}`, + ); let completed = 0; for (const document of documents) { - const imageStats = await classifyExistingImages(supabase, document.id); - await supabase - .from("documents") - .update({ - image_count: imageStats.searchable, - metadata: { - ...(document.metadata ?? {}), - searchable_image_count: imageStats.searchable, - skipped_image_count: imageStats.skipped, - image_enriched_at: new Date().toISOString(), - }, - }) - .eq("id", document.id); + const documentCoverage = coverage.get(document.id); + if (args.mode === "metadata-stamp") { + if (!documentCoverage?.summary || documentCoverage.labels.every((label) => label.source !== "generated")) { + console.log(`SKIP cannot metadata-stamp missing enrichment: ${document.file_name}`); + continue; + } + await stampExistingEnrichmentVersion({ + supabase, + document, + coverage: documentCoverage, + ragEnrichmentVersion, + }); + completed += 1; + console.log(`STAMPED ${document.file_name}`); + continue; + } + + let imageMetadata = metadataRecord(document.metadata); + let imageStats = { searchable: 0, skipped: 0, total: 0 }; + if (args.mode === "summaries-labels-images") { + imageStats = await classifyExistingImages(supabase, document.id); + imageMetadata = { + ...imageMetadata, + searchable_image_count: imageStats.searchable, + skipped_image_count: imageStats.skipped, + image_enriched_at: new Date().toISOString(), + }; + await supabase + .from("documents") + .update({ + image_count: imageStats.searchable, + metadata: imageMetadata, + }) + .eq("id", document.id); + } const evidence = await loadEvidence(supabase, document.id); if (evidence.chunks.length === 0) { @@ -238,15 +560,23 @@ async function main() { continue; } - await upsertDocumentEnrichment({ + if (args.mode === "summaries-labels-images") { + await upsertDocumentEnrichment({ + supabase, + document: { ...document, metadata: imageMetadata }, + chunks: evidence.chunks, + images: evidence.images, + }); + } + const deepMemory = await upsertDocumentDeepMemory({ supabase, - document, + document: { ...document, metadata: imageMetadata }, chunks: evidence.chunks, images: evidence.images, }); completed += 1; console.log( - `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, + `ENRICHED ${document.file_name} chunks=${evidence.chunks.length} sections=${deepMemory.sections.length} memory=${deepMemory.memoryCards.length} images=${imageStats.searchable}/${imageStats.total} skipped=${imageStats.skipped}`, ); } diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index 93cdb1836..d050c0a91 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -1,18 +1,13 @@ import { loadEnvConfig } from "@next/env"; import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { RagAnswer } from "@/lib/types"; -import { - estimateCostUsd, - findOwnerIdByEmail, - loadAdminClient, - percentile, - validateRagAnswer, -} from "./eval-utils"; +import { estimateCostUsd, findOwnerIdByEmail, loadAdminClient, percentile, validateRagAnswer } from "./eval-utils"; loadEnvConfig(process.cwd()); type EvalArgs = { ownerEmail?: string; + ownerId?: string; limit?: number; question?: string; json: boolean; @@ -44,6 +39,7 @@ type EvalResult = { function parseArgs(argv: string[]): EvalArgs { const args: EvalArgs = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, json: false, failOnThreshold: false, }; @@ -66,6 +62,7 @@ function parseArgs(argv: string[]): EvalArgs { index += 1; if (token === "--owner-email") args.ownerEmail = value; + if (token === "--owner-id") args.ownerId = value; if (token === "--limit") args.limit = Number.parseInt(value, 10); if (token === "--question") args.question = value; } @@ -90,12 +87,13 @@ function summarizeFailures(results: EvalResult[]) { const failedCases = results.filter((result) => result.failures.length > 0); const failures: string[] = []; - if (supported.length >= 18 && groundedSupported < 17) failures.push(`supported grounded ${groundedSupported}/${supported.length}`); + if (supported.length >= 18 && groundedSupported < 17) + failures.push(`supported grounded ${groundedSupported}/${supported.length}`); if (unsupported.length >= 2 && unsupportedCorrect !== unsupported.length) { failures.push(`unsupported correct ${unsupportedCorrect}/${unsupported.length}`); } if (invalidCitations > 0) failures.push(`invalid citation count ${invalidCitations}`); - if (routineExtractiveLatencies.length > 0 && percentile(routineExtractiveLatencies, 95) > 2000) { + if (routineExtractiveLatencies.length >= 5 && percentile(routineExtractiveLatencies, 95) > 2000) { failures.push("routine extractive p95 over 2000ms"); } if (complexSlow > 0) failures.push(`complex answers over 20000ms: ${complexSlow}`); @@ -147,14 +145,15 @@ function printHumanSummary(results: EvalResult[]) { console.log(` token_totals=${JSON.stringify(tokenTotals)}`); console.log( ` estimated_cost_usd=${ - estimatedCostUsd === null ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION" : estimatedCostUsd.toFixed(6) + estimatedCostUsd === null + ? "set RAG_EVAL_INPUT_USD_PER_MILLION and RAG_EVAL_OUTPUT_USD_PER_MILLION" + : estimatedCostUsd.toFixed(6) }`, ); } async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); const [{ requireOpenAIEnv, requireServerEnv }, { answerQuestionWithScope }, supabase] = await Promise.all([ import("@/lib/env"), @@ -165,11 +164,12 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); + const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public"; const cases = selectRagEvalCases({ limit: args.limit, question: args.question }); const results: EvalResult[] = []; - if (!args.json) console.log(`Running ${cases.length} RAG eval case(s).`); + if (!args.json) console.log(`Running ${cases.length} RAG eval case(s), scope=${scope}.`); for (const testCase of cases) { const answer = (await answerQuestionWithScope({ @@ -225,7 +225,7 @@ async function main() { const thresholdFailures = summarizeFailures(results); if (args.json) { - console.log(JSON.stringify({ results, thresholdFailures }, null, 2)); + console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2)); } else { printHumanSummary(results); if (thresholdFailures.length > 0) { diff --git a/scripts/eval-search.ts b/scripts/eval-search.ts index 7c16c6151..0448bd054 100644 --- a/scripts/eval-search.ts +++ b/scripts/eval-search.ts @@ -1,25 +1,37 @@ import { loadEnvConfig } from "@next/env"; +import { pathToFileURL } from "node:url"; import { buildVisualEvidence } from "@/lib/evidence"; import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { SearchResult } from "@/lib/types"; -import { expectedFileHit, findOwnerIdByEmail, hasInvalidVisualEvidence, loadAdminClient, percentile } from "./eval-utils"; +import { + expectedFileCoverage, + expectedFileHit, + findOwnerIdByEmail, + hasInvalidVisualEvidence, + loadAdminClient, + percentile, +} from "./eval-utils"; loadEnvConfig(process.cwd()); type EvalArgs = { ownerEmail?: string; + ownerId?: string; limit?: number; question?: string; json: boolean; failOnThreshold: boolean; }; -type SearchEvalResult = { +export type SearchEvalResult = { id: string; question: string; category: RagEvalCase["category"]; supported: boolean; + expectedFileCount: number; expectedHitTop3: boolean; + expectedAllHitTop5: boolean | null; + missingExpectedFiles: string[]; resultCount: number; topScore: number; topFiles: string[]; @@ -36,6 +48,7 @@ type SearchEvalResult = { function parseArgs(argv: string[]): EvalArgs { const args: EvalArgs = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, json: false, failOnThreshold: false, }; @@ -58,6 +71,7 @@ function parseArgs(argv: string[]): EvalArgs { index += 1; if (token === "--owner-email") args.ownerEmail = value; + if (token === "--owner-id") args.ownerId = value; if (token === "--limit") args.limit = Number.parseInt(value, 10); if (token === "--question") args.question = value; } @@ -76,26 +90,34 @@ function topScore(results: SearchResult[]) { function validateSearchResult(testCase: RagEvalCase, result: Omit) { const failures: string[] = []; - if (testCase.supported && !result.expectedHitTop3) failures.push("expected document not in top 3"); + if (testCase.supported && result.expectedFileCount > 1 && result.expectedAllHitTop5 === false) { + failures.push(`expected documents missing from top 5: ${result.missingExpectedFiles.join(", ")}`); + } else if (testCase.supported && result.expectedFileCount === 1 && !result.expectedHitTop3) { + failures.push("expected document not in top 3"); + } if (testCase.supported && result.resultCount === 0) failures.push("expected search results"); - if (!testCase.supported && result.topScore >= 0.82) failures.push("unsupported control produced high-confidence match"); - if (testCase.category === "routine" && result.latencyMs > 1500) failures.push("routine search over 1500ms"); - + if (!testCase.supported && result.topScore >= 0.82) + failures.push("unsupported control produced high-confidence match"); return failures; } -function summarizeFailures(results: SearchEvalResult[]) { +export function summarizeFailures(results: SearchEvalResult[]) { const routine = results.filter((result) => result.category === "routine"); const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); + const expectedCoverageCases = supported.filter((result) => result.expectedFileCount > 0); const routineLatencies = routine.map((result) => result.latencyMs); - const expectedHits = results.filter((result) => result.expectedHitTop3).length; + const expectedHits = expectedCoverageCases.filter((result) => + result.expectedFileCount > 1 ? result.expectedAllHitTop5 : result.expectedHitTop3, + ).length; const routineEmbeddingSkipped = routine.filter((result) => result.embeddingSkipped).length; const unsupportedHighConfidence = unsupported.filter((result) => result.topScore >= 0.82).length; const failures: string[] = []; - if (expectedHits < 18) failures.push(`expected document top-3 hit ${expectedHits}/${results.length}`); - if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500) failures.push("routine search p95 over 1500ms"); + if (expectedCoverageCases.length >= 18 && expectedHits < expectedCoverageCases.length) + failures.push(`expected document coverage ${expectedHits}/${expectedCoverageCases.length}`); + if (routineLatencies.length > 0 && percentile(routineLatencies, 95) > 1500) + failures.push("routine search p95 over 1500ms"); if (routine.length > 0 && routineEmbeddingSkipped / routine.length < 0.7) { failures.push(`routine embedding skipped ${routineEmbeddingSkipped}/${routine.length}`); } @@ -108,7 +130,12 @@ function summarizeFailures(results: SearchEvalResult[]) { function printHumanSummary(results: SearchEvalResult[]) { const latencies = results.map((result) => result.latencyMs); const routineLatencies = results.filter((result) => result.category === "routine").map((result) => result.latencyMs); - const expectedHits = results.filter((result) => result.expectedHitTop3).length; + const expectedCoverageCases = results.filter((result) => result.supported && result.expectedFileCount > 0); + const expectedHits = expectedCoverageCases.filter((result) => + result.expectedFileCount > 1 ? result.expectedAllHitTop5 : result.expectedHitTop3, + ).length; + const multiDocumentCoverage = expectedCoverageCases.filter((result) => result.expectedFileCount > 1); + const multiDocumentHits = multiDocumentCoverage.filter((result) => result.expectedAllHitTop5).length; const textFast = results.filter((result) => result.retrievalStrategy === "text_fast_path").length; const embeddingSkipped = results.filter((result) => result.embeddingSkipped).length; const fallbackToEmbedding = results.filter((result) => result.fallbackToEmbedding).length; @@ -120,7 +147,8 @@ function printHumanSummary(results: SearchEvalResult[]) { console.log(""); console.log("Search eval summary:"); - console.log(` expected_document_hit_top3=${expectedHits}/${results.length}`); + console.log(` expected_document_coverage=${expectedHits}/${expectedCoverageCases.length}`); + console.log(` multi_document_all_expected_top5=${multiDocumentHits}/${multiDocumentCoverage.length}`); console.log(` average_latency_ms=${Math.round(latencies.reduce((sum, value) => sum + value, 0) / results.length)}`); console.log(` p50_latency_ms=${percentile(latencies, 50)}`); console.log(` p95_latency_ms=${percentile(latencies, 95)}`); @@ -133,7 +161,6 @@ function printHumanSummary(results: SearchEvalResult[]) { async function main() { const args = parseArgs(process.argv.slice(2)); - if (!args.ownerEmail) throw new Error('Provide --owner-email "you@example.com" or set RAG_EVAL_OWNER_EMAIL.'); const [{ requireOpenAIEnv, requireServerEnv }, { searchChunksWithTelemetry }, supabase] = await Promise.all([ import("@/lib/env"), @@ -144,11 +171,12 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); + const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public"; const cases = selectRagEvalCases({ limit: args.limit, question: args.question }); const results: SearchEvalResult[] = []; - if (!args.json) console.log(`Running ${cases.length} search eval case(s).`); + if (!args.json) console.log(`Running ${cases.length} search eval case(s), scope=${scope}.`); for (const testCase of cases) { const startedAt = Date.now(); @@ -160,15 +188,21 @@ async function main() { skipCache: true, }); const latencyMs = - search.telemetry.supabase_rpc_latency_ms + search.telemetry.embedding_latency_ms + search.telemetry.rerank_latency_ms || - Date.now() - startedAt; + search.telemetry.supabase_rpc_latency_ms + + search.telemetry.embedding_latency_ms + + search.telemetry.rerank_latency_ms || Date.now() - startedAt; const visuals = buildVisualEvidence(search.results); + const top3Coverage = expectedFileCoverage(testCase.expectedFiles, search.results, 3); + const top5Coverage = expectedFileCoverage(testCase.expectedFiles, search.results, 5); const baseResult = { id: testCase.id, question: testCase.question, category: testCase.category, supported: testCase.supported, + expectedFileCount: testCase.expectedFiles.length, expectedHitTop3: expectedFileHit(testCase.expectedFiles, search.results, 3), + expectedAllHitTop5: testCase.expectedFiles.length > 1 ? top5Coverage.allHit : null, + missingExpectedFiles: testCase.expectedFiles.length > 1 ? top5Coverage.missingFiles : top3Coverage.missingFiles, resultCount: search.results.length, topScore: topScore(search.results), topFiles: Array.from(new Set(search.results.slice(0, 3).map((result) => result.file_name))), @@ -187,8 +221,10 @@ async function main() { if (!args.json) { const failureSuffix = result.failures.length ? ` FAIL=${result.failures.join("; ")}` : ""; + const expectedCoverage = + result.expectedFileCount > 1 ? ` allExpectedTop5=${result.expectedAllHitTop5}` : ""; console.log( - `SEARCH ${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} skippedEmbedding=${result.embeddingSkipped} expectedHit=${result.expectedHitTop3} topScore=${result.topScore.toFixed(3)}${failureSuffix}`, + `SEARCH ${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} skippedEmbedding=${result.embeddingSkipped} expectedHit=${result.expectedHitTop3}${expectedCoverage} topScore=${result.topScore.toFixed(3)}${failureSuffix}`, ); console.log(` Q: ${testCase.question}`); console.log(` Top files: ${result.topFiles.join("; ") || "none"}`); @@ -198,7 +234,7 @@ async function main() { const thresholdFailures = summarizeFailures(results); if (args.json) { - console.log(JSON.stringify({ results, thresholdFailures }, null, 2)); + console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2)); } else { printHumanSummary(results); if (thresholdFailures.length > 0) { @@ -209,7 +245,9 @@ async function main() { if (args.failOnThreshold && thresholdFailures.length > 0) process.exit(1); } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); + }); +} diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index fa28faa25..261ea60a8 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -30,10 +30,35 @@ export function percentile(values: number[], percentileValue: number) { return sorted[index]; } -export function expectedFileHit(expectedFiles: string[], sources: Array>, limit = 3) { - if (expectedFiles.length === 0) return true; +export type ExpectedFileCoverage = { + expectedFiles: string[]; + matchedFiles: string[]; + missingFiles: string[]; + anyHit: boolean; + allHit: boolean; +}; + +export function expectedFileCoverage( + expectedFiles: string[], + sources: Array>, + limit = 3, +): ExpectedFileCoverage { const topFiles = sources.slice(0, limit).map((source) => source.file_name.toLowerCase()); - return expectedFiles.some((expected) => topFiles.some((file) => file.includes(expected.toLowerCase()))); + const matchedFiles = expectedFiles.filter((expected) => + topFiles.some((file) => file.includes(expected.toLowerCase())), + ); + + return { + expectedFiles, + matchedFiles, + missingFiles: expectedFiles.filter((expected) => !matchedFiles.includes(expected)), + anyHit: matchedFiles.length > 0, + allHit: expectedFiles.length > 0 && matchedFiles.length === expectedFiles.length, + }; +} + +export function expectedFileHit(expectedFiles: string[], sources: Array>, limit = 3) { + return expectedFileCoverage(expectedFiles, sources, limit).anyHit; } export function hasInvalidVisualEvidence(cards: VisualEvidenceCard[] = []) { @@ -42,22 +67,32 @@ export function hasInvalidVisualEvidence(cards: VisualEvidenceCard[] = []) { export function validateRagAnswer(testCase: RagEvalCase, answer: RagAnswer) { const failures: string[] = []; - const expectedHit = expectedFileHit(testCase.expectedFiles, answer.sources); + const expectedCoverage = expectedFileCoverage( + testCase.expectedFiles, + answer.sources, + testCase.expectedFiles.length > 1 ? 5 : 3, + ); + const expectedHit = testCase.expectedFiles.length > 1 ? expectedCoverage.allHit : expectedCoverage.anyHit; const route = answer.routingMode ?? "unsupported"; const visualEvidence = answer.visualEvidence ?? []; if (testCase.supported && !answer.grounded) failures.push("expected grounded answer"); if (!testCase.supported && answer.grounded) failures.push("expected unsupported answer"); if (!testCase.allowedRoutes.includes(route)) failures.push(`unexpected route ${route}`); - if (answer.citations.length < testCase.minCitations) failures.push(`expected at least ${testCase.minCitations} citations`); - if (testCase.expectedFiles.length > 0 && !expectedHit) failures.push("expected document not in retrieved sources"); + if (answer.citations.length < testCase.minCitations) + failures.push(`expected at least ${testCase.minCitations} citations`); + if (testCase.expectedFiles.length > 1 && !expectedCoverage.allHit) { + failures.push(`expected documents missing from top 5: ${expectedCoverage.missingFiles.join(", ")}`); + } else if (testCase.expectedFiles.length === 1 && !expectedCoverage.anyHit) { + failures.push("expected document not in retrieved sources"); + } if (testCase.requireVisualEvidence && visualEvidence.length === 0) failures.push("expected visual evidence"); if (hasInvalidVisualEvidence(visualEvidence)) failures.push("decorative or zero-relevance visual evidence returned"); if (testCase.category === "complex" && (answer.latencyTimings?.total_latency_ms ?? 0) > testCase.latencyTargetMs) { failures.push(`latency over ${testCase.latencyTargetMs}ms`); } - return { expectedHit, failures }; + return { expectedHit, expectedCoverage, failures }; } export function configuredCostRates() { diff --git a/scripts/import-documents.ts b/scripts/import-documents.ts index e57569583..f9e3dc47e 100644 --- a/scripts/import-documents.ts +++ b/scripts/import-documents.ts @@ -8,8 +8,8 @@ import { type ExistingImportDocument, parseImportCliArgs, scanImportFiles, - titleFromFileName, } from "@/lib/bulk-import"; +import { planDocumentName } from "@/lib/document-naming"; loadEnvConfig(process.cwd()); @@ -118,7 +118,15 @@ async function main() { if (!files.length) return; if (args.dryRun) { let existing = new Map(); - if (args.ownerEmail) { + const dryRunOwnerId = args.ownerId ?? process.env.LOCAL_NO_AUTH_OWNER_ID; + if (dryRunOwnerId) { + const supabase = await loadAdminClient(); + existing = await existingDocumentsByHash( + supabase, + dryRunOwnerId, + Array.from(new Set(files.map((file) => file.contentHash))), + ); + } else if (args.ownerEmail) { const supabase = await loadAdminClient(); const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); existing = await existingDocumentsByHash( @@ -138,7 +146,7 @@ async function main() { ); } if (files.length > 20) console.log(`DRY RUN omitted ${files.length - 20} additional file(s).`); - if (args.ownerEmail) { + if (dryRunOwnerId || args.ownerEmail) { console.log( `DRY RUN duplicate check: would_queue=${files.length - exactCopyCount}, exact_copies=${exactCopyCount}, total=${files.length}`, ); @@ -147,11 +155,12 @@ async function main() { } const [{ env }, supabase] = await Promise.all([import("@/lib/env"), loadAdminClient()]); - if (!args.ownerEmail && !args.resume) { - throw new Error("Provide --owner-email for a new import batch."); + const configuredOwnerId = args.ownerId ?? process.env.LOCAL_NO_AUTH_OWNER_ID; + if (!configuredOwnerId && !args.ownerEmail && !args.resume) { + throw new Error("Provide --owner-id, set LOCAL_NO_AUTH_OWNER_ID, or provide --owner-email for a new import batch."); } - const requestedOwnerId = args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + const requestedOwnerId = configuredOwnerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); const batch = await loadOrCreateBatch({ supabase, ownerId: requestedOwnerId, @@ -216,16 +225,25 @@ async function main() { const documentId = createDocumentId(); const storagePath = buildImportStoragePath(ownerId, documentId, file.fileName); - const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, await readFile(file.absolutePath), { - contentType: "application/pdf", - upsert: false, + const namePlan = await planDocumentName({ + supabase: supabase as never, + ownerId, + fileName: file.fileName, + requestedTitle: null, + contentHash: file.contentHash, }); + const upload = await supabase.storage + .from(env.SUPABASE_DOCUMENT_BUCKET) + .upload(storagePath, await readFile(file.absolutePath), { + contentType: "application/pdf", + upsert: false, + }); if (upload.error) throw new Error(upload.error.message); const { error: documentError } = await supabase.from("documents").insert({ id: documentId, owner_id: ownerId, - title: titleFromFileName(file.fileName), + title: namePlan.title, description: null, file_name: file.fileName, file_type: "application/pdf", @@ -236,7 +254,7 @@ async function main() { import_batch_id: batch.id, status: "queued", metadata: { - source_title: titleFromFileName(file.fileName), + source_title: namePlan.title, publisher: null, jurisdiction: "Australia/WA", version: null, @@ -245,6 +263,12 @@ async function main() { uploaded_at: new Date().toISOString(), indexed_at: null, uploaded_by: ownerId, + original_file_name: namePlan.originalFileName, + original_title: namePlan.originalTitle, + smart_title_base: namePlan.baseTitle, + smart_title_group_key: namePlan.duplicateGroupKey, + smart_title_duplicate_index: namePlan.duplicateIndex, + smart_title_duplicate_reason: namePlan.duplicateReason, document_status: "unknown", clinical_validation_status: "unverified", extraction_quality: "unknown", diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts index 5f5e4f2f8..56e4d1913 100644 --- a/scripts/playwright-base-url.ts +++ b/scripts/playwright-base-url.ts @@ -1,9 +1,46 @@ import { execFileSync } from "node:child_process"; import path from "node:path"; +import { appName, localProjectId } from "./local-server-utils.mjs"; const projectRoot = path.resolve(__dirname, ".."); const ensureScript = path.join(projectRoot, "scripts", "ensure-local-server.mjs"); const localUrlPattern = /^http:\/\/localhost:\d+$/; +const identityScript = ` +const http = require("node:http"); +const url = process.argv[1] + "/api/local-project-id"; +const request = http.get(url, { timeout: 1500 }, (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk) => { body += chunk; }); + response.on("end", () => { + if (response.statusCode !== 200) process.exit(2); + process.stdout.write(body); + }); +}); +request.on("timeout", () => { request.destroy(); process.exit(3); }); +request.on("error", () => process.exit(4)); +`; + +function verifyLocalProjectIdentity(baseUrl: string) { + const output = execFileSync(process.execPath, ["-e", identityScript, baseUrl], { + cwd: projectRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }).trim(); + const payload = JSON.parse(output) as { + appName?: string; + projectId?: string; + localServer?: { safeLocalOrigin?: boolean }; + }; + + if ( + payload.appName !== appName || + payload.projectId !== localProjectId(projectRoot) || + payload.localServer?.safeLocalOrigin !== true + ) { + throw new Error(`Ensured URL failed /api/local-project-id guard: ${baseUrl}`); + } +} export function getPlaywrightBaseUrl() { const output = execFileSync(process.execPath, [ensureScript, "--print-url"], { @@ -16,5 +53,6 @@ export function getPlaywrightBaseUrl() { throw new Error(`Expected ensure-local-server to print a localhost URL, received: ${output || ""}`); } + verifyLocalProjectIdentity(output); return output; } diff --git a/scripts/purge-query-logs.ts b/scripts/purge-query-logs.ts new file mode 100644 index 000000000..fef3ce764 --- /dev/null +++ b/scripts/purge-query-logs.ts @@ -0,0 +1,68 @@ +import { loadEnvConfig } from "@next/env"; +import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +type PurgeArgs = { + ownerEmail?: string; + olderThanDays: number; + dryRun: boolean; +}; + +function parseArgs(argv: string[]): PurgeArgs { + const args: PurgeArgs = { + ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + olderThanDays: 90, + dryRun: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--dry-run") { + args.dryRun = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + index += 1; + + if (token === "--owner-email") args.ownerEmail = value; + if (token === "--older-than-days") args.olderThanDays = Number.parseInt(value, 10); + } + + if (!args.ownerEmail) throw new Error("Provide --owner-email."); + if (!Number.isInteger(args.olderThanDays) || args.olderThanDays <= 0) { + throw new Error("--older-than-days must be a positive integer."); + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const supabase = await loadAdminClient(); + const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail!); + const before = new Date(Date.now() - args.olderThanDays * 24 * 60 * 60 * 1000).toISOString(); + const countQuery = supabase + .from("rag_queries") + .select("id", { count: "exact", head: true }) + .eq("owner_id", ownerId) + .lt("created_at", before); + const { count, error: countError } = await countQuery; + if (countError) throw new Error(countError.message); + + console.log(`RAG query logs older than ${args.olderThanDays} day(s): ${count ?? 0}`); + if (args.dryRun || !count) return; + + const { error: deleteError } = await supabase + .from("rag_queries") + .delete() + .eq("owner_id", ownerId) + .lt("created_at", before); + if (deleteError) throw new Error(deleteError.message); + console.log(`Deleted ${count} old RAG query log(s).`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/app/api/answer/route.ts b/src/app/api/answer/route.ts index 6c77324bf..f35592b33 100644 --- a/src/app/api/answer/route.ts +++ b/src/app/api/answer/route.ts @@ -3,14 +3,15 @@ import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { answerQuestionWithScope } from "@/lib/rag"; -import { jsonError } from "@/lib/http"; -import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { consumePublicAnswerRateLimit } from "@/lib/public-rate-limit"; +import { classifyRagQuery } from "@/lib/clinical-search"; +import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; export const runtime = "nodejs"; const answerSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), }); @@ -19,25 +20,48 @@ export async function POST(request: Request) { try { const body = answerSchema.parse(await request.json()); if (isDemoMode()) { + const answer = demoAnswer(body.query, body.documentId, body.documentIds); return NextResponse.json({ - ...demoAnswer(body.query, body.documentId, body.documentIds), + ...answer, + smartApiPlan: buildSmartRagApiPlan({ + query: body.query, + queryClass: classifyRagQuery(body.query).queryClass, + results: answer.sources, + routeMode: answer.routingMode, + retrievalStrategy: "hybrid", + }), demoMode: true, }); } - const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); + const rateLimit = consumePublicAnswerRateLimit(request.headers); + if (rateLimit.limited) { + return NextResponse.json( + { error: "Too many public answer requests. Retry shortly." }, + { status: 429, headers: { "Retry-After": String(rateLimit.retryAfterSeconds) } }, + ); + } + const answer = await answerQuestionWithScope({ query: body.query, documentId: body.documentId, documentIds: body.documentIds, - ownerId: user.id, + ownerId: undefined, }); return NextResponse.json(answer); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof PublicApiError) { + return jsonError(error, error.status); + } + if (error instanceof Error) { + return jsonError( + new PublicApiError("Answer generation failed. Retry with a narrower question.", 500, { code: error.name }), + 500, + ); } - return jsonError(error, 400); + return jsonError("Answer generation failed.", 500); } } diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index 37eeec3c1..9b8ea37a5 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -2,14 +2,16 @@ import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { PublicApiError, jsonError } from "@/lib/http"; +import { consumePublicAnswerRateLimit, type PublicRateLimitResult } from "@/lib/public-rate-limit"; import { answerQuestionWithScope, type AnswerProgressEvent } from "@/lib/rag"; -import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; +import { classifyRagQuery } from "@/lib/clinical-search"; +import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; +import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; export const runtime = "nodejs"; const answerSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), }); @@ -20,6 +22,51 @@ function encodeSse(event: string, data: unknown) { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } +function rateLimitStream(rateLimit: PublicRateLimitResult) { + return new Response( + encodeSse("error", { + error: "Too many public answer requests. Retry shortly.", + status: 429, + details: { retryAfterSeconds: rateLimit.retryAfterSeconds, resetAt: rateLimit.resetAt }, + }), + { + status: 429, + headers: { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + "Retry-After": String(rateLimit.retryAfterSeconds), + }, + }, + ); +} + +function streamErrorPayload(error: unknown) { + if (error instanceof PublicApiError) { + return { message: error.message, status: error.status, details: error.details }; + } + + if (error instanceof Error) { + return { + message: "Answer generation failed. Retry with a narrower question.", + status: 503, + details: { code: error.name }, + }; + } + + return { + message: "Search processing is temporarily unavailable.", + status: 503, + }; +} + +function logStreamError(error: unknown) { + console.error("Search stream failed", { + name: error instanceof Error ? error.name : typeof error, + message: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); +} + function streamAnswer(body: AnswerBody, ownerId?: string) { const encoder = new TextEncoder(); @@ -34,7 +81,25 @@ function streamAnswer(body: AnswerBody, ownerId?: string) { try { send("progress", { stage: "retrieving", message: "Searching indexed documents." }); const answer = isDemoMode() - ? { ...demoAnswer(body.query, body.documentId, body.documentIds), demoMode: true } + ? (() => { + const demo = demoAnswer(body.query, body.documentId, body.documentIds); + const sources = annotateSearchResults(body.query, demo.sources); + const relevance = buildEvidenceRelevance(body.query, sources); + return { + ...demo, + sources, + relevance, + smartPanel: demo.smartPanel ? { ...demo.smartPanel, relevance } : demo.smartPanel, + smartApiPlan: buildSmartRagApiPlan({ + query: body.query, + queryClass: classifyRagQuery(body.query).queryClass, + results: sources, + routeMode: demo.routingMode, + retrievalStrategy: "hybrid", + }), + demoMode: true, + }; + })() : await answerQuestionWithScope({ query: body.query, documentId: body.documentId, @@ -44,11 +109,9 @@ function streamAnswer(body: AnswerBody, ownerId?: string) { }); send("final", answer); } catch (error) { - const publicError = - error instanceof PublicApiError - ? error - : new PublicApiError("Answer generation failed. Retry with a narrower question.", 500); - send("error", { error: publicError.message }); + logStreamError(error); + const streamError = streamErrorPayload(error); + send("error", { error: streamError.message, status: streamError.status, details: streamError.details }); } finally { controller.close(); } @@ -69,13 +132,20 @@ export async function POST(request: Request) { const body = answerSchema.parse(await request.json()); if (isDemoMode()) return streamAnswer(body); - const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); - return streamAnswer(body, user.id); + const rateLimit = consumePublicAnswerRateLimit(request.headers); + if (rateLimit.limited) return rateLimitStream(rateLimit); + + return streamAnswer(body); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof PublicApiError) { + return jsonError(error, error.status); + } + if (error instanceof Error) { + return jsonError(new PublicApiError("Answer processing failed.", 500, { code: error.name }), 500); } - return jsonError(error, 400); + return jsonError("Answer processing failed.", 500); } } diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 7575a4f03..181b8c83c 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { env, isDemoMode } from "@/lib/env"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; +import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { jsonError } from "@/lib/http"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -27,7 +28,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const { data: document, error: documentError } = await supabase .from("documents") - .select("id,owner_id,title,file_name,source_path,import_batch_id") + .select("id,owner_id,title,file_name,source_path,import_batch_id,metadata") .eq("id", id) .eq("owner_id", user.id) .maybeSingle(); @@ -39,17 +40,17 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const [chunksResult, imagesResult] = await Promise.all([ supabase .from("document_chunks") - .select("id,page_number,chunk_index,section_heading,content") + .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata") .eq("document_id", id) .order("chunk_index", { ascending: true }) - .limit(24), + .limit(1000), supabase .from("document_images") - .select("id,page_number,caption,image_type,labels") + .select("id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata") .eq("document_id", id) .eq("searchable", true) .order("clinical_relevance_score", { ascending: false }) - .limit(12), + .limit(200), ]); if (chunksResult.error) throw new Error(chunksResult.error.message); @@ -64,7 +65,20 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: chunks: chunksResult.data, images: imagesResult.data ?? [], }); - return NextResponse.json({ mode, enrichment }); + const deepMemory = await upsertDocumentDeepMemory({ + supabase, + document, + chunks: chunksResult.data, + images: imagesResult.data ?? [], + }); + return NextResponse.json({ + mode, + enrichment, + deepMemory: { + sectionCount: deepMemory.sections.length, + memoryCardCount: deepMemory.memoryCards.length, + }, + }); } const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: id }); diff --git a/src/app/api/documents/[id]/route.ts b/src/app/api/documents/[id]/route.ts index 05be91637..723d291a5 100644 --- a/src/app/api/documents/[id]/route.ts +++ b/src/app/api/documents/[id]/route.ts @@ -1,12 +1,163 @@ import { NextResponse } from "next/server"; +import { z } from "zod"; import { getDemoDocumentPayload } from "@/lib/demo-data"; -import { isDemoMode } from "@/lib/env"; -import { jsonError } from "@/lib/http"; +import { env, isDemoMode } from "@/lib/env"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { invalidateRagCachesForOwner } from "@/lib/rag"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; +const renameSchema = z.object({ + title: z.string().trim().min(1).max(180), +}); + +function safeMetadata(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function metadataText(metadata: Record, key: string) { + const value = metadata[key]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function compactTableText(value: string | null, limit = 500) { + if (!value) return null; + const compact = value.replace(/\s+/g, " ").trim(); + if (!compact) return null; + return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact; +} + +function metadataStringArrayRows(metadata: Record, key: string) { + const value = metadata[key]; + if (!Array.isArray(value)) return null; + const rows = value + .filter((row): row is unknown[] => Array.isArray(row)) + .map((row) => row.map((cell) => String(cell ?? "").trim())); + return rows.length ? rows : null; +} + +function metadataStringArray(metadata: Record, key: string) { + const value = metadata[key]; + if (!Array.isArray(value)) return null; + const items = value.map((item) => String(item ?? "").trim()).filter(Boolean); + return items.length ? items : null; +} + +function withImageTableMetadata(image: T) { + const metadata = safeMetadata(image.metadata); + const rawTableText = metadataText(metadata, "table_text"); + const tableText = rawTableText ?? metadataText(metadata, "table_text_snippet"); + const publicImage = { ...image }; + delete publicImage.metadata; + return { + ...publicImage, + tableLabel: metadataText(metadata, "table_label"), + tableTitle: metadataText(metadata, "table_title"), + tableRole: metadataText(metadata, "table_role"), + tableTextSnippet: compactTableText(tableText), + clinicalUseClass: metadataText(metadata, "clinical_use_class"), + clinicalUseReason: metadataText(metadata, "clinical_use_reason"), + accessibleTableMarkdown: metadataText(metadata, "accessible_table_markdown") ?? rawTableText, + tableRows: metadataStringArrayRows(metadata, "table_rows"), + tableColumns: metadataStringArray(metadata, "table_columns"), + }; +} + +function storageWarningsFrom(error: unknown, label: string) { + const message = + error && typeof error === "object" && "message" in error ? String(error.message) : "Storage cleanup failed."; + return `${label}: ${message}`; +} + +async function removeStorageObjects(args: { + supabase: ReturnType; + sourcePath: string | null; + imagePaths: string[]; +}) { + const warnings: string[] = []; + let storageRemoved = 0; + + if (args.sourcePath) { + const sourceRemove = await args.supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([args.sourcePath]); + if (sourceRemove.error) { + warnings.push(storageWarningsFrom(sourceRemove.error, "Source PDF")); + } else { + storageRemoved += sourceRemove.data?.length ?? 0; + } + } + + const uniqueImagePaths = Array.from(new Set(args.imagePaths.filter(Boolean))); + for (let start = 0; start < uniqueImagePaths.length; start += 1000) { + const paths = uniqueImagePaths.slice(start, start + 1000); + const imageRemove = await args.supabase.storage.from(env.SUPABASE_IMAGE_BUCKET).remove(paths); + if (imageRemove.error) { + warnings.push(storageWarningsFrom(imageRemove.error, "Extracted images")); + } else { + storageRemoved += imageRemove.data?.length ?? 0; + } + } + + return { storageRemoved, storageWarnings: warnings }; +} + +async function createStorageCleanupJob(args: { + supabase: ReturnType; + ownerId: string; + documentId: string; + documentTitle: string; + sourcePath: string | null; + imagePaths: string[]; +}) { + const { data, error } = await args.supabase + .from("storage_cleanup_jobs") + .insert({ + owner_id: args.ownerId, + document_id: args.documentId, + document_title: args.documentTitle, + document_bucket: env.SUPABASE_DOCUMENT_BUCKET, + document_paths: args.sourcePath ? [args.sourcePath] : [], + image_bucket: env.SUPABASE_IMAGE_BUCKET, + image_paths: Array.from(new Set(args.imagePaths.filter(Boolean))), + status: "pending", + metadata: { + operation: "permanent_document_delete", + created_by: "api/documents/[id]", + }, + }) + .select("id") + .single(); + + if (error) throw new Error(error.message); + return data.id as string; +} + +async function updateStorageCleanupJob(args: { + supabase: ReturnType; + cleanupJobId: string; + status: "completed" | "failed"; + storageRemoved: number; + warnings: string[]; +}) { + const { error } = await args.supabase + .from("storage_cleanup_jobs") + .update({ + status: args.status, + attempts: 1, + storage_removed: args.storageRemoved, + last_error: args.warnings.length ? args.warnings.join("; ") : null, + completed_at: args.status === "completed" ? new Date().toISOString() : null, + metadata: { + operation: "permanent_document_delete", + storage_warnings: args.warnings, + }, + }) + .eq("id", args.cleanupJobId); + + return error ? storageWarningsFrom(error, "Cleanup ledger") : null; +} + export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { try { const { id } = await params; @@ -43,10 +194,11 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: const { data: images, error: imagesError } = await supabase .from("document_images") .select( - "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels", + "id,page_number,storage_path,caption,bbox,mime_type,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata", ) .eq("document_id", id) - .eq("searchable", true) + .neq("image_type", "logo_decorative") + .or("searchable.eq.true,source_kind.eq.table_crop") .order("page_number", { ascending: true }); if (imagesError) throw new Error(imagesError.message); @@ -79,7 +231,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: summary: summaryResult.data ?? null, }, pages: pages ?? [], - images: images ?? [], + images: (images ?? []).map(withImageTableMetadata), chunks: chunks ?? [], }); } catch (error) { @@ -89,3 +241,167 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: return jsonError(error); } } + +export async function PATCH(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + if (isDemoMode()) { + return NextResponse.json({ error: "Demo documents cannot be renamed." }, { status: 400 }); + } + + const parsed = renameSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) { + throw new PublicApiError("Enter a document title between 1 and 180 characters."); + } + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const { data: document, error: documentError } = await supabase + .from("documents") + .select("id,owner_id,title,file_name,storage_path,content_hash,metadata") + .eq("id", id) + .eq("owner_id", user.id) + .maybeSingle(); + + if (documentError) throw new Error(documentError.message); + if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 }); + if (document.title.trim() === parsed.data.title) { + throw new PublicApiError("Document title is unchanged."); + } + + const metadata = safeMetadata(document.metadata); + const { data: updated, error: updateError } = await supabase + .from("documents") + .update({ + title: parsed.data.title, + metadata: { + ...metadata, + renamed_at: new Date().toISOString(), + previous_title: document.title, + original_file_name: metadata.original_file_name ?? document.file_name, + }, + }) + .eq("id", id) + .eq("owner_id", user.id) + .select("*") + .single(); + + if (updateError) throw new Error(updateError.message); + invalidateRagCachesForOwner(user.id); + return NextResponse.json({ document: updated }); + } catch (error) { + if (error instanceof AuthenticationError) { + return unauthorizedResponse(); + } + return jsonError(error); + } +} + +export async function DELETE(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + if (isDemoMode()) { + return NextResponse.json({ error: "Demo documents cannot be deleted." }, { status: 400 }); + } + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const { data: document, error: documentError } = await supabase + .from("documents") + .select("id,owner_id,title,storage_path") + .eq("id", id) + .eq("owner_id", user.id) + .maybeSingle(); + + if (documentError) throw new Error(documentError.message); + if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 }); + + const { data: activeJobs, error: activeJobsError } = await supabase + .from("ingestion_jobs") + .select("id,status") + .eq("document_id", id) + .eq("status", "processing") + .limit(1); + + if (activeJobsError) throw new Error(activeJobsError.message); + if ((activeJobs ?? []).length > 0) { + throw new PublicApiError("Document is currently indexing. Stop or wait for the worker before deleting.", 409); + } + + const [{ data: images, error: imagesError }, { data: chunks, error: chunksError }] = await Promise.all([ + supabase.from("document_images").select("storage_path").eq("document_id", id), + supabase.from("document_chunks").select("id").eq("document_id", id), + ]); + + if (imagesError) throw new Error(imagesError.message); + if (chunksError) throw new Error(chunksError.message); + + const chunkIds = (chunks ?? []).map((chunk) => chunk.id).filter(Boolean); + const imagePaths = (images ?? []).map((image) => image.storage_path).filter(Boolean); + const cleanupJobId = await createStorageCleanupJob({ + supabase, + ownerId: user.id, + documentId: id, + documentTitle: document.title, + sourcePath: document.storage_path, + imagePaths, + }); + + if (chunkIds.length > 0) { + const { error: queryDeleteError } = await supabase + .from("rag_queries") + .delete() + .eq("owner_id", user.id) + .overlaps("source_chunk_ids", chunkIds); + if (queryDeleteError) { + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: "failed", + storageRemoved: 0, + warnings: [`Query log delete: ${queryDeleteError.message}`], + }); + throw new Error(ledgerWarning ? `${queryDeleteError.message}; ${ledgerWarning}` : queryDeleteError.message); + } + } + + const { error: deleteError } = await supabase.from("documents").delete().eq("id", id).eq("owner_id", user.id); + if (deleteError) { + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: "failed", + storageRemoved: 0, + warnings: [`Database delete: ${deleteError.message}`], + }); + throw new Error(ledgerWarning ? `${deleteError.message}; ${ledgerWarning}` : deleteError.message); + } + + const cleanup = await removeStorageObjects({ + supabase, + sourcePath: document.storage_path, + imagePaths, + }); + const ledgerWarning = await updateStorageCleanupJob({ + supabase, + cleanupJobId, + status: cleanup.storageWarnings.length > 0 ? "failed" : "completed", + storageRemoved: cleanup.storageRemoved, + warnings: cleanup.storageWarnings, + }); + if (ledgerWarning) cleanup.storageWarnings.push(ledgerWarning); + + invalidateRagCachesForOwner(user.id); + return NextResponse.json({ + deleted: true, + documentId: id, + storageRemoved: cleanup.storageRemoved, + storageWarnings: cleanup.storageWarnings, + }); + } catch (error) { + if (error instanceof AuthenticationError) { + return unauthorizedResponse(); + } + return jsonError(error); + } +} diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c88fb2898..95ce5eb21 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -7,49 +7,189 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f export const runtime = "nodejs"; +const DOCUMENT_LIST_COLUMNS = [ + "id", + "owner_id", + "title", + "description", + "file_name", + "file_type", + "file_size", + "storage_path", + "content_hash", + "source_path", + "import_batch_id", + "status", + "page_count", + "chunk_count", + "image_count", + "error_message", + "metadata", + "created_at", + "updated_at", +].join(","); + +const LABEL_LIST_COLUMNS = [ + "id", + "document_id", + "owner_id", + "label", + "label_type", + "source", + "confidence", + "metadata", + "created_at", + "updated_at", +].join(","); + +const SUMMARY_LIST_COLUMNS = [ + "id", + "document_id", + "owner_id", + "summary", + "clinical_specifics", + "source_chunk_ids", + "source_image_ids", + "model", + "metadata", + "generated_at", + "created_at", + "updated_at", +].join(","); + +const VALID_STATUSES = new Set(["queued", "processing", "indexed", "failed"]); +const ACTIVE_DOCUMENT_STATUSES = new Set(["queued", "processing"]); +const ACTIVE_INDEXING_POLL_MS = 5_000; + +type DocumentListRow = Record & { id: string; status?: string | null }; +type LabelListRow = Record & { document_id: string }; +type SummaryListRow = Record & { document_id: string }; + +function parsePositiveInt(value: string | null, fallback: number, max: number) { + const parsed = Number.parseInt(value ?? "", 10); + if (!Number.isInteger(parsed) || parsed <= 0) return fallback; + return Math.min(parsed, max); +} + +function parseOffset(value: string | null) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 0; +} + +function ilikePattern(value: string) { + return `%${value.replace(/[%_]/g, "\\$&")}%`; +} + +function safeSearchTerm(value: string) { + return value + .replace(/[,%()]/g, " ") + .replace(/\s+/g, " ") + .trim() + .slice(0, 120); +} + +function indexingState(documents: DocumentListRow[]) { + const active = documents.some((document) => ACTIVE_DOCUMENT_STATUSES.has(String(document.status ?? ""))); + return { + active, + pollAfterMs: active ? ACTIVE_INDEXING_POLL_MS : null, + }; +} + +function documentsResponse(payload: Record, indexing: ReturnType) { + return NextResponse.json( + { + ...payload, + indexing, + }, + { + headers: { + "Cache-Control": "private, no-store", + "X-Indexing-Active": String(indexing.active), + "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""), + }, + }, + ); +} + export async function GET(request: Request) { try { if (isDemoMode()) { - return NextResponse.json({ documents: demoDocuments, demoMode: true }); + return documentsResponse({ documents: demoDocuments, demoMode: true }, { active: false, pollAfterMs: null }); } + const url = new URL(request.url); + const limit = parsePositiveInt(url.searchParams.get("limit"), 100, 200); + const offset = parseOffset(url.searchParams.get("offset")); + const search = safeSearchTerm(url.searchParams.get("q") ?? ""); + const status = url.searchParams.get("status")?.trim() ?? ""; + const includeMeta = url.searchParams.get("includeMeta") !== "false"; + const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); - const { data, error } = await supabase + let query = supabase .from("documents") - .select("*") + .select(DOCUMENT_LIST_COLUMNS, { count: "exact" }) .eq("owner_id", user.id) - .order("created_at", { ascending: false }); + .order("created_at", { ascending: false }) + .range(offset, offset + limit - 1); + + if (status && VALID_STATUSES.has(status)) { + query = query.eq("status", status); + } + if (search) { + const pattern = ilikePattern(search); + query = query.or(`title.ilike.${pattern},file_name.ilike.${pattern}`); + } + + const { data, error, count } = await query; if (error) throw new Error(error.message); - const documents = data ?? []; + const documents = (data ?? []) as unknown as DocumentListRow[]; const documentIds = documents.map((document) => document.id); + const indexing = indexingState(documents); - if (documentIds.length === 0) return NextResponse.json({ documents }); + const pagination = { + limit, + offset, + total: count ?? documents.length, + nextOffset: offset + documents.length, + hasMore: count === null ? documents.length === limit : offset + documents.length < count, + }; + + if (documentIds.length === 0 || !includeMeta) { + return documentsResponse({ documents, pagination }, indexing); + } const [labelsResult, summariesResult] = await Promise.all([ - supabase.from("document_labels").select("*").in("document_id", documentIds), - supabase.from("document_summaries").select("*").in("document_id", documentIds), + supabase.from("document_labels").select(LABEL_LIST_COLUMNS).in("document_id", documentIds), + supabase.from("document_summaries").select(SUMMARY_LIST_COLUMNS).in("document_id", documentIds), ]); if (labelsResult.error) throw new Error(labelsResult.error.message); if (summariesResult.error) throw new Error(summariesResult.error.message); const labelsByDocument = new Map(); - for (const label of labelsResult.data ?? []) { + for (const label of (labelsResult.data ?? []) as unknown as LabelListRow[]) { const existing = labelsByDocument.get(label.document_id) ?? []; existing.push(label); labelsByDocument.set(label.document_id, existing); } - const summariesByDocument = new Map((summariesResult.data ?? []).map((summary) => [summary.document_id, summary])); - - return NextResponse.json({ - documents: documents.map((document) => ({ - ...document, - labels: labelsByDocument.get(document.id) ?? [], - summary: summariesByDocument.get(document.id) ?? null, - })), - }); + const summariesByDocument = new Map( + ((summariesResult.data ?? []) as unknown as SummaryListRow[]).map((summary) => [summary.document_id, summary]), + ); + + return documentsResponse( + { + documents: documents.map((document) => ({ + ...document, + labels: labelsByDocument.get(document.id) ?? [], + summary: summariesByDocument.get(document.id) ?? null, + })), + pagination, + }, + indexing, + ); } catch (error) { if (error instanceof AuthenticationError) { return unauthorizedResponse(); diff --git a/src/app/api/ingestion/batches/route.ts b/src/app/api/ingestion/batches/route.ts index 4d2a2b1e0..a57c3386c 100644 --- a/src/app/api/ingestion/batches/route.ts +++ b/src/app/api/ingestion/batches/route.ts @@ -6,9 +6,41 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f export const runtime = "nodejs"; +const ACTIVE_BATCH_STATUSES = new Set(["queued", "processing"]); +const ACTIVE_INDEXING_POLL_MS = 5_000; + +type BatchRow = Record & { status?: string | null }; + +function batchesIndexingState(batches: BatchRow[]) { + const activeBatchCount = batches.filter((batch) => ACTIVE_BATCH_STATUSES.has(String(batch.status ?? ""))).length; + return { + activeBatchCount, + hasActiveBatches: activeBatchCount > 0, + pollAfterMs: activeBatchCount > 0 ? ACTIVE_INDEXING_POLL_MS : null, + }; +} + +function batchesResponse(batches: BatchRow[], extra: Record = {}) { + const indexing = batchesIndexingState(batches); + return NextResponse.json( + { + batches, + ...indexing, + ...extra, + }, + { + headers: { + "Cache-Control": "private, no-store", + "X-Indexing-Active": String(indexing.hasActiveBatches), + "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""), + }, + }, + ); +} + export async function GET(request: Request) { try { - if (isDemoMode()) return NextResponse.json({ batches: [], demoMode: true }); + if (isDemoMode()) return batchesResponse([], { demoMode: true }); const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); @@ -20,7 +52,7 @@ export async function GET(request: Request) { .limit(20); if (error) throw new Error(error.message); - return NextResponse.json({ batches: data ?? [] }); + return batchesResponse((data ?? []) as unknown as BatchRow[]); } catch (error) { if (error instanceof AuthenticationError) return unauthorizedResponse(); return jsonError(error); diff --git a/src/app/api/ingestion/jobs/route.ts b/src/app/api/ingestion/jobs/route.ts index c7e4fd85e..6a9ab2064 100644 --- a/src/app/api/ingestion/jobs/route.ts +++ b/src/app/api/ingestion/jobs/route.ts @@ -6,9 +6,41 @@ import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } f export const runtime = "nodejs"; +const ACTIVE_JOB_STATUSES = new Set(["pending", "processing"]); +const ACTIVE_INDEXING_POLL_MS = 5_000; + +type JobRow = Record & { status?: string | null }; + +function jobsIndexingState(jobs: JobRow[]) { + const activeJobCount = jobs.filter((job) => ACTIVE_JOB_STATUSES.has(String(job.status ?? ""))).length; + return { + activeJobCount, + hasActiveJobs: activeJobCount > 0, + pollAfterMs: activeJobCount > 0 ? ACTIVE_INDEXING_POLL_MS : null, + }; +} + +function jobsResponse(jobs: JobRow[], extra: Record = {}) { + const indexing = jobsIndexingState(jobs); + return NextResponse.json( + { + jobs, + ...indexing, + ...extra, + }, + { + headers: { + "Cache-Control": "private, no-store", + "X-Indexing-Active": String(indexing.hasActiveJobs), + "X-Poll-After-Ms": String(indexing.pollAfterMs ?? ""), + }, + }, + ); +} + export async function GET(request: Request) { try { - if (isDemoMode()) return NextResponse.json({ jobs: [], demoMode: true }); + if (isDemoMode()) return jobsResponse([], { demoMode: true }); const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); @@ -25,7 +57,7 @@ export async function GET(request: Request) { const { data, error } = await query; if (error) throw new Error(error.message); - return NextResponse.json({ jobs: data ?? [] }); + return jobsResponse((data ?? []) as unknown as JobRow[]); } catch (error) { if (error instanceof AuthenticationError) return unauthorizedResponse(); return jsonError(error); diff --git a/src/app/api/local-project-id/route.ts b/src/app/api/local-project-id/route.ts index 57316373f..555742d0e 100644 --- a/src/app/api/local-project-id/route.ts +++ b/src/app/api/local-project-id/route.ts @@ -1,11 +1,12 @@ -import { appName, localProjectId } from "../../../../scripts/local-server-utils.mjs"; +import { localProjectRequestIdentityPayload } from "@/lib/local-project-guard"; export const dynamic = "force-dynamic"; export const runtime = "nodejs"; -export async function GET() { - return Response.json({ - appName, - projectId: localProjectId(process.cwd()), +export async function GET(request: Request) { + return Response.json(localProjectRequestIdentityPayload(request), { + headers: { + "Cache-Control": "no-store", + }, }); } diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 87aefd48d..5e1c152e3 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -3,77 +3,249 @@ import { z } from "zod"; import { demoSearch } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { buildSmartPanel, buildVisualEvidence, diversifySearchResults } from "@/lib/evidence"; -import { fetchRelatedDocuments } from "@/lib/document-enrichment"; -import { jsonError } from "@/lib/http"; +import { annotateDocumentMatches, annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; +import { fetchRelatedDocuments, toDocumentMatch } from "@/lib/document-enrichment"; +import { jsonError, PublicApiError } from "@/lib/http"; +import { isClinicalImageEvidence } from "@/lib/image-filtering"; import { searchChunksWithTelemetry } from "@/lib/rag"; +import { classifyRagQuery } from "@/lib/clinical-search"; +import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { createAdminClient } from "@/lib/supabase/admin"; -import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; +import { consumePublicSearchRateLimit } from "@/lib/public-rate-limit"; +import type { SearchResult } from "@/lib/types"; export const runtime = "nodejs"; const searchSchema = z.object({ - query: z.string().trim().min(2), + query: z.string().trim().min(1), topK: z.number().int().min(1).max(20).optional(), documentId: z.string().uuid().optional(), documentIds: z.array(z.string().uuid()).max(25).optional(), + mode: z.enum(["answer", "documents"]).optional().default("answer"), + documentLimit: z.number().int().min(1).max(50).optional().default(20), includeRelatedDocuments: z.boolean().optional().default(true), }); +type SearchRequestBody = z.infer; + +const publicSearchInflight = new Map>(); + +function publicSearchKey(body: SearchRequestBody) { + return JSON.stringify({ + query: body.query.toLowerCase().replace(/\s+/g, " ").trim(), + topK: body.topK ?? null, + documentId: body.documentId ?? null, + documentIds: body.documentIds?.length ? [...body.documentIds].sort() : [], + mode: body.mode, + documentLimit: body.documentLimit, + includeRelatedDocuments: body.includeRelatedDocuments, + }); +} + +async function coalescePublicSearch>(key: string, producer: () => Promise) { + const existing = publicSearchInflight.get(key) as Promise | undefined; + if (existing) return { payload: await existing, coalesced: true }; + + const pending = producer().finally(() => { + publicSearchInflight.delete(key); + }); + publicSearchInflight.set(key, pending); + return { payload: await pending, coalesced: false }; +} + +function buildDocumentMatchesFromResults(results: SearchResult[], limit: number) { + const grouped = new Map< + string, + { + document_id: string; + title: string; + file_name: string; + bestPages: number[]; + bestChunkIds: string[]; + imageCount: number; + tableCount: number; + score: number; + } + >(); + for (const result of results) { + const current = grouped.get(result.document_id); + const score = result.hybrid_score ?? result.similarity; + const page = result.page_number ?? null; + const clinicalImages = result.images?.filter((image) => isClinicalImageEvidence(image)) ?? []; + const tableCount = clinicalImages.filter((image) => image.source_kind === "table_crop").length; + const imageCount = clinicalImages.length; + if (!current) { + grouped.set(result.document_id, { + document_id: result.document_id, + title: result.title, + file_name: result.file_name, + bestPages: page ? [page] : [], + bestChunkIds: [result.id], + imageCount, + tableCount, + score, + }); + continue; + } + current.score = Math.max(current.score, score); + if (page && !current.bestPages.includes(page)) current.bestPages.push(page); + if (!current.bestChunkIds.includes(result.id)) current.bestChunkIds.push(result.id); + current.imageCount += imageCount; + current.tableCount += tableCount; + } + + return Array.from(grouped.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map((document) => ({ + ...document, + labels: [], + summarySnippet: null, + matchReason: `Matched ${document.bestChunkIds.length} indexed passage${ + document.bestChunkIds.length === 1 ? "" : "s" + }`, + })); +} + +async function buildPublicSearchPayload(body: SearchRequestBody) { + const supabase = createAdminClient(); + const search = await searchChunksWithTelemetry({ + query: body.query, + topK: body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8), + documentId: body.documentId, + documentIds: body.documentIds, + ownerId: undefined, + }); + const resultLimit = + body.mode === "documents" ? Math.max(body.topK ?? 12, Math.min(20, body.documentLimit)) : (body.topK ?? 8); + const results = annotateSearchResults(body.query, diversifySearchResults(search.results, resultLimit, 4, true)); + + const relatedDocuments = body.includeRelatedDocuments + ? await fetchRelatedDocuments({ + supabase, + ownerId: undefined, + query: body.query, + results, + limit: body.mode === "documents" ? body.documentLimit : undefined, + }) + : []; + const smartPanel = buildSmartPanel(body.query, results); + const relevance = buildEvidenceRelevance(body.query, results); + const documentMatches = + body.mode === "documents" + ? annotateDocumentMatches(body.query, relatedDocuments.map(toDocumentMatch), results) + : []; + const queryClass = search.telemetry.query_class ?? classifyRagQuery(body.query).queryClass; + const smartApiPlan = buildSmartRagApiPlan({ + query: body.query, + queryClass, + results, + retrievalStrategy: search.telemetry.retrieval_strategy, + routeMode: body.mode === "documents" ? undefined : "fast", + preferredResponseMode: body.mode === "documents" ? "document_lookup" : undefined, + }); + + return { + results, + visualEvidence: buildVisualEvidence(results), + relevance, + relatedDocuments, + documentMatches, + smartPanel: { ...smartPanel, relevance, relatedDocuments }, + smartApiPlan, + telemetry: { + query_class: queryClass, + relevance_verdict: relevance.verdict, + relevance_score: relevance.score, + direct_source_count: relevance.directSourceCount, + weak_source_count: relevance.weakSourceCount, + retrieval_strategy: search.telemetry.retrieval_strategy, + smart_api_intent: smartApiPlan.intent, + smart_api_response_mode: smartApiPlan.responseMode, + smart_api_source_link_count: smartApiPlan.sourceLinkCount, + search_cache_hit: search.telemetry.search_cache_hit, + embedding_skipped: search.telemetry.embedding_skipped, + embedding_cache_hit: search.telemetry.embedding_cache_hit, + text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms, + embedding_latency_ms: search.telemetry.embedding_latency_ms, + supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms, + rerank_latency_ms: search.telemetry.rerank_latency_ms, + memory_card_count: search.telemetry.memory_card_count, + memory_top_score: search.telemetry.memory_top_score, + weighted_top_score: search.telemetry.weighted_top_score, + rrf_top_score: search.telemetry.rrf_top_score, + }, + }; +} + export async function POST(request: Request) { try { const body = searchSchema.parse(await request.json()); if (isDemoMode()) { - const results = demoSearch(body.query, body.topK ?? 8, body.documentId, body.documentIds); + const results = annotateSearchResults( + body.query, + demoSearch(body.query, body.topK ?? 8, body.documentId, body.documentIds), + ); + const queryClass = classifyRagQuery(body.query).queryClass; + const relevance = buildEvidenceRelevance(body.query, results); + const documentMatches = + body.mode === "documents" + ? annotateDocumentMatches(body.query, buildDocumentMatchesFromResults(results, body.documentLimit), results) + : []; return NextResponse.json({ results, visualEvidence: buildVisualEvidence(results), - smartPanel: buildSmartPanel(body.query, results), + relevance, + smartPanel: { ...buildSmartPanel(body.query, results), relevance }, + smartApiPlan: buildSmartRagApiPlan({ + query: body.query, + queryClass, + results, + retrievalStrategy: "hybrid", + routeMode: body.mode === "documents" ? undefined : "fast", + preferredResponseMode: body.mode === "documents" ? "document_lookup" : undefined, + }), relatedDocuments: [], + documentMatches, demoMode: true, }); } - const supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); - const search = await searchChunksWithTelemetry({ - query: body.query, - topK: body.topK ?? 8, - documentId: body.documentId, - documentIds: body.documentIds, - ownerId: user.id, - }); - const results = diversifySearchResults(search.results, body.topK ?? 8); - - const relatedDocuments = body.includeRelatedDocuments - ? await fetchRelatedDocuments({ - supabase, - ownerId: user.id, - query: body.query, - results, - }) - : []; - const smartPanel = buildSmartPanel(body.query, results); + const rateLimit = consumePublicSearchRateLimit(request.headers); + if (rateLimit.limited) { + return NextResponse.json( + { + error: "Search is temporarily rate limited because too many requests were received. Retry shortly.", + retryAfterSeconds: rateLimit.retryAfterSeconds, + }, + { + status: 429, + headers: { + "Retry-After": String(rateLimit.retryAfterSeconds), + }, + }, + ); + } + const key = publicSearchKey(body); + const { payload, coalesced } = await coalescePublicSearch(key, () => buildPublicSearchPayload(body)); return NextResponse.json({ - results, - visualEvidence: buildVisualEvidence(results), - relatedDocuments, - smartPanel: { ...smartPanel, relatedDocuments }, + ...payload, telemetry: { - retrieval_strategy: search.telemetry.retrieval_strategy, - search_cache_hit: search.telemetry.search_cache_hit, - embedding_skipped: search.telemetry.embedding_skipped, - embedding_cache_hit: search.telemetry.embedding_cache_hit, - text_fast_path_latency_ms: search.telemetry.text_fast_path_latency_ms, - embedding_latency_ms: search.telemetry.embedding_latency_ms, - supabase_rpc_latency_ms: search.telemetry.supabase_rpc_latency_ms, - rerank_latency_ms: search.telemetry.rerank_latency_ms, + ...payload.telemetry, + coalesced, }, }); } catch (error) { - if (error instanceof AuthenticationError) { - return unauthorizedResponse(); + if (error instanceof z.ZodError) { + return jsonError(error, 400); + } + if (error instanceof Error && error.message.trim()) { + return jsonError( + new PublicApiError("Search failed. Retry with a narrower question.", 500, { code: error.name }), + 500, + ); } - return jsonError(error, 400); + return jsonError(error, 500); } } diff --git a/src/app/api/setup-status/route.ts b/src/app/api/setup-status/route.ts index 02b855f2d..3ab615dd1 100644 --- a/src/app/api/setup-status/route.ts +++ b/src/app/api/setup-status/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { env, isDemoMode } from "@/lib/env"; +import { localProjectRequestIdentityPayload, unsafeLocalProjectResponse } from "@/lib/local-project-guard"; import { createAdminClient } from "@/lib/supabase/admin"; import { checkSupabaseProjectConfig, formatSupabaseProjectCheck } from "@/lib/supabase/project"; @@ -16,15 +17,37 @@ type SetupCheck = { detail: string; }; +type WorkerStatus = { + check: SetupCheck; + activeWork: boolean; +}; + +type SetupStatusPayload = { + demoMode: boolean; + checks: SetupCheck[]; + indexingActive: boolean; + pollAfterMs: number | null; + generatedAt: string; +}; + +type AdminClient = ReturnType; + const requiredSupabaseEnvPresent = Boolean(env.NEXT_PUBLIC_SUPABASE_URL && env.SUPABASE_SERVICE_ROLE_KEY); const supabaseProjectCheck = checkSupabaseProjectConfig(env, { requireMetadata: true }); const supabaseProjectCanBeQueried = requiredSupabaseEnvPresent && supabaseProjectCheck.status !== "mismatch"; +const ACTIVE_INDEXING_POLL_MS = Math.max(3_000, Math.min(env.WORKER_POLL_MS, 15_000)); +const SETUP_RECHECK_POLL_MS = 60_000; +const SETUP_STATUS_ACTIVE_CACHE_MS = Math.max(2_000, Math.min(ACTIVE_INDEXING_POLL_MS, 5_000)); +const SETUP_STATUS_IDLE_CACHE_MS = 30_000; + +let setupStatusCache: { expiresAt: number; payload: SetupStatusPayload } | null = null; +let setupStatusInFlight: Promise | null = null; function check(id: SetupCheckId, label: string, status: SetupCheckStatus, detail: string): SetupCheck { return { id, label, status, detail }; } -async function readSchemaStatus() { +async function readSchemaStatus(supabase: AdminClient | null) { if (!requiredSupabaseEnvPresent) { return check( "schema", @@ -44,7 +67,9 @@ async function readSchemaStatus() { } try { - const supabase = createAdminClient(); + if (!supabase) { + throw new Error("Supabase admin client is unavailable."); + } const [documents, jobs, batches, buckets] = await Promise.all([ supabase.from("documents").select("id,content_hash,import_batch_id").limit(1), supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts,locked_at").limit(1), @@ -82,94 +107,177 @@ async function readSchemaStatus() { } } -async function readWorkerStatus() { +async function readWorkerStatus(supabase: AdminClient | null): Promise { const label = "npm run worker running"; if (!requiredSupabaseEnvPresent) { - return check( - "worker", - label, - "unknown", - "Worker status cannot be inferred until Supabase is configured.", - ); + return { + check: check("worker", label, "unknown", "Worker status cannot be inferred until Supabase is configured."), + activeWork: false, + }; } if (!supabaseProjectCanBeQueried) { - return check( - "worker", - label, - "unknown", - "Worker status cannot be inferred while Supabase points at the wrong project.", - ); + return { + check: check( + "worker", + label, + "unknown", + "Worker status cannot be inferred while Supabase points at the wrong project.", + ), + activeWork: false, + }; } try { - const supabase = createAdminClient(); + if (!supabase) { + throw new Error("Supabase admin client is unavailable."); + } const [latestResult, activeResult] = await Promise.all([ supabase.from("ingestion_jobs").select("status,updated_at").order("updated_at", { ascending: false }).limit(1), - supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]), + supabase + .from("ingestion_jobs") + .select("id", { count: "exact", head: true }) + .in("status", ["pending", "processing"]), ]); if (latestResult.error || activeResult.error) { - return check("worker", label, "unknown", "Ingestion jobs could not be checked."); + return { + check: check("worker", label, "unknown", "Ingestion jobs could not be checked."), + activeWork: false, + }; } const latest = latestResult.data?.[0]; + const activeWork = (activeResult.count ?? 0) > 0; if (!latest?.updated_at) { - return check("worker", label, "unknown", "No ingestion activity has been recorded yet."); + return { + check: check("worker", label, "unknown", "No ingestion activity has been recorded yet."), + activeWork, + }; } const updatedAt = new Date(latest.updated_at).getTime(); const recentWindowMs = Math.max(env.WORKER_POLL_MS * 6, 60_000); if (Number.isFinite(updatedAt) && Date.now() - updatedAt <= recentWindowMs) { - return check("worker", label, "ready", "Recent ingestion activity was detected."); + return { + check: check("worker", label, "ready", "Recent ingestion activity was detected."), + activeWork, + }; } if ((activeResult.count ?? 0) === 0 && latest.status === "completed") { - return check("worker", label, "ready", "No queued ingestion work is waiting; the latest job completed."); + return { + check: check("worker", label, "ready", "No queued ingestion work is waiting; the latest job completed."), + activeWork: false, + }; } - return check( - "worker", - label, - "unknown", - "Queued or processing ingestion work exists, but no recent activity proves the worker is active.", - ); + return { + check: check( + "worker", + label, + "unknown", + "Queued or processing ingestion work exists, but no recent activity proves the worker is active.", + ), + activeWork, + }; } catch { - return check("worker", label, "unknown", "Worker status could not be inferred."); + return { + check: check("worker", label, "unknown", "Worker status could not be inferred."), + activeWork: false, + }; } } -export async function GET() { - const [schema, worker] = await Promise.all([readSchemaStatus(), readWorkerStatus()]); +function setupStatusCacheTtl(payload: SetupStatusPayload) { + return payload.indexingActive ? SETUP_STATUS_ACTIVE_CACHE_MS : SETUP_STATUS_IDLE_CACHE_MS; +} + +function setupStatusResponse(payload: SetupStatusPayload) { + return NextResponse.json(payload, { + headers: { + "Cache-Control": "private, max-age=5, stale-while-revalidate=30", + "X-Poll-After-Ms": String(payload.pollAfterMs ?? ""), + }, + }); +} + +async function buildSetupStatusPayload(): Promise { + const supabase = supabaseProjectCanBeQueried ? createAdminClient() : null; + const [schema, worker] = await Promise.all([readSchemaStatus(supabase), readWorkerStatus(supabase)]); - return NextResponse.json({ + const checks = [ + check( + "env", + ".env.local configured", + requiredSupabaseEnvPresent ? "ready" : "needs_setup", + requiredSupabaseEnvPresent + ? "Required Supabase server environment variables are present." + : "Set the required Supabase URL and server key.", + ), + check( + "project", + "Clinical KB Database target", + supabaseProjectCheck.status === "ready" ? "ready" : "needs_setup", + formatSupabaseProjectCheck(supabaseProjectCheck), + ), + schema, + check( + "openai", + "OpenAI API key available", + env.OPENAI_API_KEY ? "ready" : "needs_setup", + env.OPENAI_API_KEY + ? "OPENAI_API_KEY is present for answers, embeddings, and captions." + : "Set OPENAI_API_KEY before real indexing or answers.", + ), + worker.check, + ]; + const setupSettled = checks.every((item) => item.status === "ready"); + const pollAfterMs = worker.activeWork ? ACTIVE_INDEXING_POLL_MS : setupSettled ? null : SETUP_RECHECK_POLL_MS; + + return { demoMode: isDemoMode(), - checks: [ - check( - "env", - ".env.local configured", - requiredSupabaseEnvPresent ? "ready" : "needs_setup", - requiredSupabaseEnvPresent - ? "Required Supabase server environment variables are present." - : "Set the required Supabase URL and server key.", - ), - check( - "project", - "Clinical KB Database target", - supabaseProjectCheck.status === "ready" ? "ready" : "needs_setup", - formatSupabaseProjectCheck(supabaseProjectCheck), - ), - schema, - check( - "openai", - "OpenAI API key available", - env.OPENAI_API_KEY ? "ready" : "needs_setup", - env.OPENAI_API_KEY - ? "OPENAI_API_KEY is present for answers, embeddings, and captions." - : "Set OPENAI_API_KEY before real indexing or answers.", - ), - worker, - ], + checks, + indexingActive: worker.activeWork, + pollAfterMs, + generatedAt: new Date().toISOString(), + }; +} + +async function readSetupStatusPayload() { + const now = Date.now(); + if (setupStatusCache && setupStatusCache.expiresAt > now) { + return setupStatusCache.payload; + } + + if (setupStatusInFlight) { + return setupStatusInFlight; + } + + const promise = buildSetupStatusPayload().then((payload) => { + setupStatusCache = { + expiresAt: Date.now() + setupStatusCacheTtl(payload), + payload, + }; + return payload; }); + setupStatusInFlight = promise; + + try { + return await promise; + } finally { + if (setupStatusInFlight === promise) { + setupStatusInFlight = null; + } + } +} + +export async function GET(request: Request) { + const identity = localProjectRequestIdentityPayload(request); + if (!identity.localServer.safeLocalOrigin) { + return unsafeLocalProjectResponse(identity); + } + + return setupStatusResponse(await readSetupStatusPayload()); } diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index 9ebbf0b4b..e26238659 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { NextResponse } from "next/server"; import { env } from "@/lib/env"; import { assertAllowedFile, jsonError } from "@/lib/http"; +import { planDocumentName, type SupabaseLike } from "@/lib/document-naming"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -54,7 +55,15 @@ export async function POST(request: Request) { if (upload.error) throw new Error(upload.error.message); uploadedPath = storagePath; - const title = String(formData.get("title") || file.name.replace(/\.[^.]+$/, "")); + const namingSupabase = supabase as unknown as SupabaseLike; + const namePlan = await planDocumentName({ + supabase: namingSupabase, + ownerId: user.id, + fileName: file.name, + requestedTitle: formData.get("title") ? String(formData.get("title")) : null, + contentHash, + }); + const title = namePlan.title; const description = formData.get("description") ? String(formData.get("description")) : null; const uploadedAt = new Date().toISOString(); @@ -81,6 +90,12 @@ export async function POST(request: Request) { uploaded_at: uploadedAt, indexed_at: null, uploaded_by: user.id, + original_file_name: namePlan.originalFileName, + original_title: namePlan.originalTitle, + smart_title_base: namePlan.baseTitle, + smart_title_group_key: namePlan.duplicateGroupKey, + smart_title_duplicate_index: namePlan.duplicateIndex, + smart_title_duplicate_reason: namePlan.duplicateReason, document_status: "unknown", clinical_validation_status: "unverified", extraction_quality: "unknown", diff --git a/src/app/layout.tsx b/src/app/layout.tsx index c9e586c6d..78b847a04 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -1,6 +1,5 @@ import type { Metadata, Viewport } from "next"; import { Geist, Geist_Mono } from "next/font/google"; -import Script from "next/script"; import { AuthProvider } from "@/lib/supabase/client"; import "./globals.css"; @@ -30,16 +29,6 @@ export default function RootLayout({ }: Readonly<{ children: React.ReactNode; }>) { - const themeScript = ` - (() => { - try { - const stored = localStorage.getItem("clinical-kb-theme"); - const dark = stored === "dark" || (!stored && window.matchMedia("(prefers-color-scheme: dark)").matches); - document.documentElement.classList.toggle("dark", dark); - } catch {} - })(); - `; - return ( -