diff --git a/.env.example b/.env.example index 5b213fd68..bbee23e7c 100644 --- a/.env.example +++ b/.env.example @@ -22,4 +22,9 @@ CHUNK_SIZE=2000 CHUNK_OVERLAP=200 WORKER_POLL_MS=5000 WORKER_BATCH_SIZE=24 +WORKER_CONCURRENCY=2 +WORKER_MAX_ATTEMPTS=3 +WORKER_STALE_AFTER_MINUTES=45 PYTHON_BIN=python +# Optional when Tesseract is installed outside PATH on Windows. +TESSERACT_CMD=C:\Program Files\Tesseract-OCR\tesseract.exe diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index dda37b3c9..f5c384572 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -9,6 +9,7 @@ on: permissions: contents: read + pull-requests: read security-events: write jobs: diff --git a/package-lock.json b/package-lock.json index 938a92613..5ad5a38fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@next/env": "^16.2.6", - "@supabase/supabase-js": "^2.105.4", + "@supabase/supabase-js": "^2.106.1", "exceljs": "^4.4.0", "jszip": "^3.10.1", "lucide-react": "^1.16.0", @@ -1125,9 +1125,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1144,9 +1141,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1163,9 +1157,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1182,9 +1173,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1201,9 +1189,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1220,9 +1205,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1239,9 +1221,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1258,9 +1237,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1277,9 +1253,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1302,9 +1275,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1327,9 +1297,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1352,9 +1319,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1377,9 +1341,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1402,9 +1363,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1427,9 +1385,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1452,9 +1407,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1691,9 +1643,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1710,9 +1659,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1729,9 +1675,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1748,9 +1691,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1767,9 +1707,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1883,9 +1820,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1902,9 +1836,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1921,9 +1852,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1940,9 +1868,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2159,9 +2084,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2179,9 +2101,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2199,9 +2118,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2219,9 +2135,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2239,9 +2152,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2259,9 +2169,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2382,9 +2289,9 @@ "license": "MIT" }, "node_modules/@supabase/auth-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.105.4.tgz", - "integrity": "sha512-Ejfa37M5xoIwoxVebxRahnwubPo8g22qkXQ4p50+N9MIvU9UZoN+A8dwVPtczzGf8oV/YXN80ZPxK4aWXuSN/A==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.106.1.tgz", + "integrity": "sha512-7eyheXfAGwkB9bZewJPs+N3UYt6kra2JG6mIxNEgbkvcO15PLD1e75PTIUEYYl3zrifm3GrpShVl7QZxKrXO/w==", "license": "MIT", "dependencies": { "tslib": "2.8.1" @@ -2394,9 +2301,9 @@ } }, "node_modules/@supabase/functions-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.105.4.tgz", - "integrity": "sha512-JVNKbBft3Qkja+WlGaE026AJ2AH9K0UTsxsfvEIHgd4zFrBor4BYRCrYFrv9IDsvVqkF72wKDsODJl5GY/C4tA==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.106.1.tgz", + "integrity": "sha512-XbOPnR2mW7jp/EcW447xmGwCa+/Wc00Hkw8t4tUIJjRsHQ4xAESsLKcyLRhRJjJoUnJVXUlC+w0wUxUCM7CG2A==", "license": "MIT", "dependencies": { "tslib": "2.8.1" @@ -2412,9 +2319,9 @@ "license": "MIT" }, "node_modules/@supabase/postgrest-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.105.4.tgz", - "integrity": "sha512-SppIyLo/kTwIlz1qpv2HN1EQqBg0GVktrDDFsXygYROha3MgVn4rT7p5EjFHFqXQm2rdRGb/BI7bc+jr10m91w==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.106.1.tgz", + "integrity": "sha512-Qbn6d2lqiqeaBX1Uko0e/hL90dtQGRN6CG2wMVQtJpRFstlVW45qmUTyTOsiB8dYUWu1fWYo4YzJuDbokGv3tQ==", "license": "MIT", "dependencies": { "tslib": "2.8.1" @@ -2424,9 +2331,9 @@ } }, "node_modules/@supabase/realtime-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.105.4.tgz", - "integrity": "sha512-6ov6c59+8D9h7q4M4Gy/uDJlC0Akxl9/714Y+6vJ+Sijuc16TS/p5DwhfRCLNcIhNiej1gEt+CQUwsjiPt4PxQ==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.106.1.tgz", + "integrity": "sha512-eQCYri5E8KsjpDgC7g28cOOS2britjUWdNSJluFMainqrMRepzjOnaxqXc3RoAz7H0dxmBrfLUNF6NGP8C+YaA==", "license": "MIT", "dependencies": { "@supabase/phoenix": "^0.4.2", @@ -2437,9 +2344,9 @@ } }, "node_modules/@supabase/storage-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.105.4.tgz", - "integrity": "sha512-Jx+pzMP1Whjof2PWHoVBUA75/p7PQE9CqKBzn1oXVyJDOggMLSH2OzVWwsXYaxEpdC1K/KltwmOX44nL3LHl9g==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.106.1.tgz", + "integrity": "sha512-HWcLIhqinhWKpOQ3WzglR2unjW0eh9J7yOu3IZrZNIEkraK4La/HDvTqndljGsNw0itPtyHhuKBxRoPG1VUARw==", "license": "MIT", "dependencies": { "iceberg-js": "^0.8.1", @@ -2450,16 +2357,16 @@ } }, "node_modules/@supabase/supabase-js": { - "version": "2.105.4", - "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.105.4.tgz", - "integrity": "sha512-cEnx+k49knU+qdIP7rXwR6fqEXPHZs+74xFK1R0S8MgQ7v9tbePVdGxvO03n3bPympMdJWVLadARBfU4TgNHCQ==", + "version": "2.106.1", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.106.1.tgz", + "integrity": "sha512-gP4HurGkGu7Z3xoOCjtAI17BKKp7jpsmwY0Ssbsks9XQRzJ7ZhK7LxfLdBSYgUdgZCQgjRK+Mr7+cl4Gxrk0Rw==", "license": "MIT", "dependencies": { - "@supabase/auth-js": "2.105.4", - "@supabase/functions-js": "2.105.4", - "@supabase/postgrest-js": "2.105.4", - "@supabase/realtime-js": "2.105.4", - "@supabase/storage-js": "2.105.4" + "@supabase/auth-js": "2.106.1", + "@supabase/functions-js": "2.106.1", + "@supabase/postgrest-js": "2.106.1", + "@supabase/realtime-js": "2.106.1", + "@supabase/storage-js": "2.106.1" }, "engines": { "node": ">=20.0.0" @@ -2607,9 +2514,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2627,9 +2531,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2647,9 +2548,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2667,9 +2565,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2709,6 +2604,72 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.1", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.0.tgz", @@ -3258,9 +3219,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3275,9 +3233,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3292,9 +3247,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3309,9 +3261,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3326,9 +3275,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3343,9 +3289,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3360,9 +3303,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3377,9 +3317,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6875,9 +6812,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6899,9 +6833,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6923,9 +6854,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6947,9 +6875,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7965,9 +7890,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -7988,9 +7910,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -8011,9 +7930,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -8034,9 +7950,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -8057,9 +7970,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ diff --git a/package.json b/package.json index a785ff945..5e80b9caa 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,15 @@ "format": "prettier --write .", "format:check": "prettier --check .", "worker": "tsx worker/index.ts", + "worker:once": "tsx worker/index.ts --once", + "import:docs": "tsx scripts/import-documents.ts", + "check:indexing": "tsx scripts/check-indexing.ts", "samples": "tsx scripts/generate-sample-documents.ts", "samples:check": "tsx scripts/check-sample-extraction.ts" }, "dependencies": { "@next/env": "^16.2.6", - "@supabase/supabase-js": "^2.105.4", + "@supabase/supabase-js": "^2.106.1", "exceljs": "^4.4.0", "jszip": "^3.10.1", "lucide-react": "^1.16.0", diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts new file mode 100644 index 000000000..7efdeb6ee --- /dev/null +++ b/scripts/check-indexing.ts @@ -0,0 +1,36 @@ +import { loadEnvConfig } from "@next/env"; + +loadEnvConfig(process.cwd()); + +async function main() { + const [{ env, requireOpenAIEnv, requireServerEnv }, { createAdminClient }, { checkPythonPdfPrerequisites }] = + await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("../worker/prerequisites"), + ]); + + requireServerEnv(); + requireOpenAIEnv(); + + const prereqs = await checkPythonPdfPrerequisites(); + if (!prereqs.ok) { + throw new Error(`PDF/OCR prerequisite check failed: ${prereqs.detail}`); + } + + const supabase = createAdminClient(); + const { error: batchError } = await supabase.from("import_batches").select("id").limit(1); + if (batchError) throw new Error(batchError.message); + const { error: jobError } = await supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts").limit(1); + if (jobError) throw new Error(jobError.message); + + console.log("Indexing prerequisites ready."); + console.log("Supabase bulk ingestion tables are reachable."); + console.log(`Embedding model: ${env.OPENAI_EMBEDDING_MODEL}`); + console.log(`Worker concurrency: ${env.WORKER_CONCURRENCY}`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/scripts/import-documents.ts b/scripts/import-documents.ts new file mode 100644 index 000000000..e57569583 --- /dev/null +++ b/scripts/import-documents.ts @@ -0,0 +1,297 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { loadEnvConfig } from "@next/env"; +import { + buildImportStoragePath, + createDocumentId, + formatExactDuplicateSkip, + type ExistingImportDocument, + parseImportCliArgs, + scanImportFiles, + titleFromFileName, +} from "@/lib/bulk-import"; + +loadEnvConfig(process.cwd()); + +type SupabaseAdmin = Awaited>; + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) { + const normalized = email.trim().toLowerCase(); + const perPage = 1000; + + for (let page = 1; page < 50; page += 1) { + const { data, error } = await supabase.auth.admin.listUsers({ page, perPage }); + if (error) throw new Error(error.message); + const user = data.users.find((candidate) => candidate.email?.toLowerCase() === normalized); + if (user?.id) return user.id; + if (data.users.length < perPage) break; + } + + throw new Error(`No Supabase Auth user found for ${email}. Sign in once before importing.`); +} + +async function existingDocumentsByHash(supabase: SupabaseAdmin, ownerId: string, hashes: string[]) { + const existing = new Map(); + for (let start = 0; start < hashes.length; start += 100) { + const hashBatch = hashes.slice(start, start + 100); + const { data, error } = await supabase + .from("documents") + .select("id,storage_path,title,source_path,content_hash") + .eq("owner_id", ownerId) + .in("content_hash", hashBatch); + + if (error) throw new Error(error.message); + for (const document of data ?? []) { + if (document.content_hash) existing.set(document.content_hash, document); + } + } + return existing; +} + +async function loadOrCreateBatch(args: { + supabase: SupabaseAdmin; + ownerId?: string; + name: string; + sourceRoot: string; + include: string; + limit?: number; + resume?: string; + totalFiles: number; + totalBytes: number; + dryRun: boolean; +}) { + if (args.dryRun) { + return { id: "dry-run", name: args.name, owner_id: null }; + } + + if (args.resume) { + const { data, error } = await args.supabase + .from("import_batches") + .select("id,name,owner_id") + .eq("id", args.resume) + .maybeSingle(); + if (error) throw new Error(error.message); + if (!data) throw new Error(`Import batch not found: ${args.resume}`); + if (args.ownerId && data.owner_id !== args.ownerId) throw new Error("Resume batch belongs to a different user."); + return data; + } + + if (!args.ownerId) throw new Error("New import batches require an owner user."); + + const { data, error } = await args.supabase + .from("import_batches") + .insert({ + owner_id: args.ownerId, + name: args.name, + source_root: args.sourceRoot, + include_glob: args.include, + status: "processing", + total_files: args.totalFiles, + total_bytes: args.totalBytes, + metadata: { + importer: "local-folder", + document_scope: "guidelines-only", + limit: args.limit ?? null, + }, + }) + .select("id,name,owner_id") + .single(); + + if (error) throw new Error(error.message); + return data; +} + +async function main() { + const args = parseImportCliArgs(process.argv.slice(2)); + const root = path.resolve(args.path); + const files = await scanImportFiles(root, args.include, args.limit); + const totalBytes = files.reduce((sum, file) => sum + file.size, 0); + const batchName = args.batchName ?? `Clinical guideline import ${new Date().toISOString().slice(0, 10)}`; + + console.log(`Scanned ${files.length} file(s), ${(totalBytes / 1024 / 1024).toFixed(1)} MB total.`); + + if (!files.length) return; + if (args.dryRun) { + let existing = new Map(); + if (args.ownerEmail) { + const supabase = await loadAdminClient(); + const ownerId = await findOwnerIdByEmail(supabase, args.ownerEmail); + existing = await existingDocumentsByHash( + supabase, + ownerId, + Array.from(new Set(files.map((file) => file.contentHash))), + ); + } + + const exactCopyCount = files.filter((file) => existing.has(file.contentHash)).length; + for (const file of files.slice(0, 20)) { + const duplicate = existing.get(file.contentHash); + console.log( + duplicate + ? formatExactDuplicateSkip(file, duplicate, { dryRun: true }) + : `DRY RUN ${file.relativePath} ${file.contentHash.slice(0, 12)}`, + ); + } + if (files.length > 20) console.log(`DRY RUN omitted ${files.length - 20} additional file(s).`); + if (args.ownerEmail) { + console.log( + `DRY RUN duplicate check: would_queue=${files.length - exactCopyCount}, exact_copies=${exactCopyCount}, total=${files.length}`, + ); + } + return; + } + + const [{ env }, supabase] = await Promise.all([import("@/lib/env"), loadAdminClient()]); + if (!args.ownerEmail && !args.resume) { + throw new Error("Provide --owner-email for a new import batch."); + } + + const requestedOwnerId = args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined; + const batch = await loadOrCreateBatch({ + supabase, + ownerId: requestedOwnerId, + name: batchName, + sourceRoot: root, + include: args.include, + limit: args.limit, + resume: args.resume, + totalFiles: files.length, + totalBytes, + dryRun: args.dryRun, + }); + const ownerId = requestedOwnerId ?? batch.owner_id; + if (!ownerId) throw new Error("Import batch has no owner user."); + + const existing = await existingDocumentsByHash( + supabase, + ownerId, + Array.from(new Set(files.map((file) => file.contentHash))), + ); + + let queued = 0; + let skipped = 0; + let failed = 0; + + for (const file of files) { + try { + const duplicate = existing.get(file.contentHash); + if (duplicate && !args.force) { + skipped += 1; + console.log(formatExactDuplicateSkip(file, duplicate)); + continue; + } + + if (duplicate && args.force) { + await supabase.rpc("reset_document_index", { p_document_id: duplicate.id }); + await supabase + .from("documents") + .update({ + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + source_path: file.absolutePath, + import_batch_id: batch.id, + }) + .eq("id", duplicate.id); + const { error: jobError } = await supabase.from("ingestion_jobs").insert({ + document_id: duplicate.id, + batch_id: batch.id, + status: "pending", + stage: "queued", + progress: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS, + }); + if (jobError) throw new Error(jobError.message); + queued += 1; + console.log(`REQUEUE ${file.relativePath}`); + continue; + } + + const documentId = createDocumentId(); + const storagePath = buildImportStoragePath(ownerId, documentId, file.fileName); + const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, await readFile(file.absolutePath), { + contentType: "application/pdf", + upsert: false, + }); + if (upload.error) throw new Error(upload.error.message); + + const { error: documentError } = await supabase.from("documents").insert({ + id: documentId, + owner_id: ownerId, + title: titleFromFileName(file.fileName), + description: null, + file_name: file.fileName, + file_type: "application/pdf", + file_size: file.size, + storage_path: storagePath, + content_hash: file.contentHash, + source_path: file.absolutePath, + import_batch_id: batch.id, + status: "queued", + metadata: { + source_title: titleFromFileName(file.fileName), + publisher: null, + jurisdiction: "Australia/WA", + version: null, + publication_date: null, + review_date: null, + uploaded_at: new Date().toISOString(), + indexed_at: null, + uploaded_by: ownerId, + document_status: "unknown", + clinical_validation_status: "unverified", + extraction_quality: "unknown", + confidentiality_scope: "guidelines-only", + source_path: file.absolutePath, + content_hash: file.contentHash, + import_batch_id: batch.id, + }, + }); + if (documentError) throw new Error(documentError.message); + + const { error: jobError } = await supabase.from("ingestion_jobs").insert({ + document_id: documentId, + batch_id: batch.id, + status: "pending", + stage: "queued", + progress: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS, + }); + if (jobError) throw new Error(jobError.message); + + queued += 1; + console.log(`QUEUE ${file.relativePath}`); + } catch (error) { + failed += 1; + console.error(`FAIL ${file.relativePath}`, error instanceof Error ? error.message : String(error)); + } + } + + const status = failed > 0 ? "completed_with_errors" : "completed"; + await supabase + .from("import_batches") + .update({ + status, + queued_files: queued, + skipped_files: skipped, + failed_files: failed, + completed_at: new Date().toISOString(), + }) + .eq("id", batch.id); + + console.log( + `Import batch ${batch.id}: queued=${queued}, exact_copies_skipped=${skipped}, failed=${failed}, total=${files.length}`, + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts new file mode 100644 index 000000000..be2e7c7ec --- /dev/null +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -0,0 +1,56 @@ +import { NextResponse } from "next/server"; +import { env, isDemoMode } from "@/lib/env"; +import { jsonError } from "@/lib/http"; +import { createAdminClient } from "@/lib/supabase/admin"; +import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; + +export const runtime = "nodejs"; + +export async function POST(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + if (isDemoMode()) return NextResponse.json({ error: "Reindex is unavailable in demo mode." }, { status: 400 }); + + const { id } = await params; + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + + const { data: document, error: documentError } = await supabase + .from("documents") + .select("id,import_batch_id") + .eq("id", id) + .eq("owner_id", user.id) + .maybeSingle(); + + if (documentError) throw new Error(documentError.message); + if (!document) return NextResponse.json({ error: "Document not found." }, { status: 404 }); + + const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: id }); + if (resetError) throw new Error(resetError.message); + + const { error: updateError } = await supabase + .from("documents") + .update({ status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 }) + .eq("id", id) + .eq("owner_id", user.id); + if (updateError) throw new Error(updateError.message); + + const { data: job, error: jobError } = await supabase + .from("ingestion_jobs") + .insert({ + document_id: id, + batch_id: document.import_batch_id ?? null, + status: "pending", + stage: "queued", + progress: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS, + }) + .select() + .single(); + + if (jobError) throw new Error(jobError.message); + return NextResponse.json({ job }, { status: 201 }); + } catch (error) { + if (error instanceof AuthenticationError) return unauthorizedResponse(); + return jsonError(error, 400); + } +} diff --git a/src/app/api/ingestion/batches/route.ts b/src/app/api/ingestion/batches/route.ts new file mode 100644 index 000000000..4d2a2b1e0 --- /dev/null +++ b/src/app/api/ingestion/batches/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; +import { isDemoMode } from "@/lib/env"; +import { jsonError } from "@/lib/http"; +import { createAdminClient } from "@/lib/supabase/admin"; +import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; + +export const runtime = "nodejs"; + +export async function GET(request: Request) { + try { + if (isDemoMode()) return NextResponse.json({ batches: [], demoMode: true }); + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const { data, error } = await supabase + .from("import_batches") + .select("*") + .eq("owner_id", user.id) + .order("created_at", { ascending: false }) + .limit(20); + + if (error) throw new Error(error.message); + return NextResponse.json({ batches: data ?? [] }); + } catch (error) { + if (error instanceof AuthenticationError) return unauthorizedResponse(); + return jsonError(error); + } +} diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts new file mode 100644 index 000000000..89a5487d5 --- /dev/null +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -0,0 +1,61 @@ +import { NextResponse } from "next/server"; +import { env, isDemoMode } from "@/lib/env"; +import { jsonError } from "@/lib/http"; +import { createAdminClient } from "@/lib/supabase/admin"; +import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; + +export const runtime = "nodejs"; + +export async function POST(request: Request, { params }: { params: Promise<{ id: string }> }) { + try { + if (isDemoMode()) return NextResponse.json({ error: "Retry is unavailable in demo mode." }, { status: 400 }); + + const { id } = await params; + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + + const { data: job, error: jobError } = await supabase + .from("ingestion_jobs") + .select("id,document_id,batch_id,documents!inner(owner_id)") + .eq("id", id) + .eq("documents.owner_id", user.id) + .maybeSingle(); + + if (jobError) throw new Error(jobError.message); + if (!job) return NextResponse.json({ error: "Ingestion job not found." }, { status: 404 }); + + const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: job.document_id }); + if (resetError) throw new Error(resetError.message); + + const { error: documentError } = await supabase + .from("documents") + .update({ status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 }) + .eq("id", job.document_id) + .eq("owner_id", user.id); + if (documentError) throw new Error(documentError.message); + + const { data, error } = await supabase + .from("ingestion_jobs") + .update({ + status: "pending", + stage: "queued", + progress: 0, + error_message: null, + attempt_count: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS, + locked_at: null, + locked_by: null, + next_run_at: new Date().toISOString(), + completed_at: null, + }) + .eq("id", id) + .select() + .single(); + + if (error) throw new Error(error.message); + return NextResponse.json({ job: data }); + } catch (error) { + if (error instanceof AuthenticationError) return unauthorizedResponse(); + return jsonError(error, 400); + } +} diff --git a/src/app/api/ingestion/jobs/route.ts b/src/app/api/ingestion/jobs/route.ts new file mode 100644 index 000000000..c7e4fd85e --- /dev/null +++ b/src/app/api/ingestion/jobs/route.ts @@ -0,0 +1,33 @@ +import { NextResponse } from "next/server"; +import { isDemoMode } from "@/lib/env"; +import { jsonError } from "@/lib/http"; +import { createAdminClient } from "@/lib/supabase/admin"; +import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; + +export const runtime = "nodejs"; + +export async function GET(request: Request) { + try { + if (isDemoMode()) return NextResponse.json({ jobs: [], demoMode: true }); + + const supabase = createAdminClient(); + const user = await requireAuthenticatedUser(request, supabase); + const batchId = new URL(request.url).searchParams.get("batchId"); + + let query = supabase + .from("ingestion_jobs") + .select("*, documents!inner(title,file_name,status,owner_id)") + .eq("documents.owner_id", user.id) + .order("created_at", { ascending: false }) + .limit(100); + + if (batchId) query = query.eq("batch_id", batchId); + + const { data, error } = await query; + if (error) throw new Error(error.message); + return NextResponse.json({ jobs: data ?? [] }); + } catch (error) { + if (error instanceof AuthenticationError) return unauthorizedResponse(); + return jsonError(error); + } +} diff --git a/src/app/api/setup-status/route.ts b/src/app/api/setup-status/route.ts index 829b695ff..7703e7d38 100644 --- a/src/app/api/setup-status/route.ts +++ b/src/app/api/setup-status/route.ts @@ -33,9 +33,10 @@ async function readSchemaStatus() { try { const supabase = createAdminClient(); - const [documents, jobs, buckets] = await Promise.all([ - supabase.from("documents").select("id").limit(1), - supabase.from("ingestion_jobs").select("id").limit(1), + const [documents, jobs, batches, buckets] = await Promise.all([ + supabase.from("documents").select("id,content_hash,import_batch_id").limit(1), + supabase.from("ingestion_jobs").select("id,attempt_count,max_attempts,locked_at").limit(1), + supabase.from("import_batches").select("id").limit(1), supabase.storage.listBuckets(), ]); @@ -44,7 +45,7 @@ async function readSchemaStatus() { buckets.data?.some((bucket) => bucket.id === env.SUPABASE_DOCUMENT_BUCKET) && buckets.data?.some((bucket) => bucket.id === env.SUPABASE_IMAGE_BUCKET); - if (documents.error || jobs.error || !hasRequiredBuckets) { + if (documents.error || jobs.error || batches.error || !hasRequiredBuckets) { return check( "schema", "supabase/schema.sql applied", @@ -70,10 +71,12 @@ async function readSchemaStatus() { } async function readWorkerStatus() { + const label = "npm run worker running"; + if (!requiredSupabaseEnvPresent) { return check( "worker", - "npm run worker running", + label, "unknown", "Worker status cannot be inferred until Supabase is configured.", ); @@ -81,35 +84,38 @@ async function readWorkerStatus() { try { const supabase = createAdminClient(); - const { data, error } = await supabase - .from("ingestion_jobs") - .select("status,updated_at") - .order("updated_at", { ascending: false }) - .limit(1); - - if (error) { - return check("worker", "npm run worker running", "unknown", "Ingestion jobs could not be checked."); + const [latestResult, activeResult] = await Promise.all([ + supabase.from("ingestion_jobs").select("status,updated_at").order("updated_at", { ascending: false }).limit(1), + supabase.from("ingestion_jobs").select("id", { count: "exact", head: true }).in("status", ["pending", "processing"]), + ]); + + if (latestResult.error || activeResult.error) { + return check("worker", label, "unknown", "Ingestion jobs could not be checked."); } - const latest = data?.[0]; + const latest = latestResult.data?.[0]; if (!latest?.updated_at) { - return check("worker", "npm run worker running", "unknown", "No ingestion activity has been recorded yet."); + return check("worker", label, "unknown", "No ingestion activity has been recorded yet."); } const updatedAt = new Date(latest.updated_at).getTime(); const recentWindowMs = Math.max(env.WORKER_POLL_MS * 6, 60_000); if (Number.isFinite(updatedAt) && Date.now() - updatedAt <= recentWindowMs) { - return check("worker", "npm run worker running", "ready", "Recent ingestion activity was detected."); + return check("worker", label, "ready", "Recent ingestion activity was detected."); + } + + if ((activeResult.count ?? 0) === 0 && latest.status === "completed") { + return check("worker", label, "ready", "No queued ingestion work is waiting; the latest job completed."); } return check( "worker", - "npm run worker running", + label, "unknown", - "No recent ingestion activity proves the worker is active.", + "Queued or processing ingestion work exists, but no recent activity proves the worker is active.", ); } catch { - return check("worker", "npm run worker running", "unknown", "Worker status could not be inferred."); + return check("worker", label, "unknown", "Worker status could not be inferred."); } } diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index 2fe5e2516..9ebbf0b4b 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { NextResponse } from "next/server"; import { env } from "@/lib/env"; import { assertAllowedFile, jsonError } from "@/lib/http"; @@ -26,6 +27,24 @@ export async function POST(request: Request) { const safeName = file.name.replace(/[^\w.\-() ]+/g, "_"); const storagePath = `${user.id}/documents/${documentId}/${safeName}`; const buffer = Buffer.from(await file.arrayBuffer()); + const contentHash = createHash("sha256").update(buffer).digest("hex"); + + const { data: duplicate, error: duplicateError } = await supabase + .from("documents") + .select("id,title,file_name,status,page_count,chunk_count,image_count,created_at") + .eq("owner_id", user.id) + .eq("content_hash", contentHash) + .maybeSingle(); + + if (duplicateError) throw new Error(duplicateError.message); + if (duplicate?.id) { + return NextResponse.json({ + document: duplicate, + duplicate: true, + duplicateReason: "exact_content_hash", + message: `Exact copy already exists as "${duplicate.title}"; no duplicate job was queued.`, + }); + } const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, buffer, { contentType: file.type, @@ -50,6 +69,7 @@ export async function POST(request: Request) { file_type: file.type, file_size: file.size, storage_path: storagePath, + content_hash: contentHash, status: "queued", metadata: { source_title: title, @@ -66,6 +86,7 @@ export async function POST(request: Request) { extraction_quality: "unknown", max_upload_mb: env.MAX_UPLOAD_MB, confidentiality_scope: "guidelines-only", + content_hash: contentHash, }, }) .select() @@ -77,9 +98,11 @@ export async function POST(request: Request) { .from("ingestion_jobs") .insert({ document_id: documentId, + batch_id: null, status: "pending", stage: "queued", progress: 0, + max_attempts: env.WORKER_MAX_ATTEMPTS, }) .select() .single(); diff --git a/src/app/globals.css b/src/app/globals.css index 8802ca2d1..9212fe755 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -5,7 +5,6 @@ /* Theme tokens */ :root { --background: #eef6f7; - --foreground: #0f172a; --app-shell: #04141b; --app-shell-muted: #0b2a35; --app-shell-accent: #0e8078; @@ -52,7 +51,6 @@ .dark { --background: #03080d; - --foreground: #e5edf4; --app-shell: #020d14; --app-shell-muted: #08232d; --app-shell-accent: #2bc9bb; @@ -97,14 +95,6 @@ color-scheme: dark; } -/* Tailwind theme bridge */ -@theme inline { - --color-background: var(--background); - --color-foreground: var(--foreground); - --font-sans: var(--font-geist-sans); - --font-mono: var(--font-geist-mono); -} - /* Reset and base document styles */ * { box-sizing: border-box; @@ -123,7 +113,7 @@ body { radial-gradient(circle at 8% 0%, color-mix(in srgb, var(--primary) 9%, transparent), transparent 28rem), radial-gradient(circle at 98% 12%, color-mix(in srgb, var(--info) 7%, transparent), transparent 26rem), var(--background); - color: var(--foreground); + color: var(--text); font-family: var(--font-geist-sans), Arial, Helvetica, sans-serif; font-feature-settings: "liga" 1, @@ -247,7 +237,6 @@ summary::-webkit-details-marker { :root, .dark { --background: Canvas; - --foreground: CanvasText; --app-shell: CanvasText; --surface: Canvas; --surface-raised: Canvas; diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 0fc6898c2..092a806e2 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -21,6 +21,7 @@ import { Mail, Moon, Quote, + RefreshCw, Search, ShieldAlert, Sun, @@ -71,6 +72,7 @@ import { nextTheme, resolveThemePreference, type ResolvedTheme } from "@/lib/the import type { ClinicalDocument, BestSourceRecommendation, + ImportBatch, IngestionJob, QuoteCard, RagAnswer, @@ -549,7 +551,7 @@ function MasterSearchHeader({
-

Clinical Guide

+

Clinical Guide

{demoMode && ( Demo data @@ -599,7 +601,7 @@ function MasterSearchHeader({ placeholder="Ask a guideline question" className={cn( "w-full rounded-lg border border-white/20 bg-white/95 pl-12 pr-12 font-semibold text-slate-950 shadow-[0_16px_34px_rgb(0_0_0_/_14%),inset_0_1px_0_rgb(255_255_255_/_82%)] outline-none transition placeholder:text-slate-500 focus:border-[color:var(--focus)] focus:ring-4 focus:ring-teal-300/25 dark:bg-slate-950/90 dark:text-slate-50 dark:placeholder:text-slate-500", - compactMobile ? "h-12 text-sm sm:h-14 sm:text-base" : "h-14 text-base", + "h-11 text-sm sm:text-base", )} /> {query && ( @@ -623,7 +625,7 @@ function MasterSearchHeader({ ? "Enter at least two characters to ask" : "Generate a source-backed answer" } - className={cn(primaryControl, compactMobile ? "h-12 rounded-lg px-3 sm:h-14 sm:px-5" : "h-14 rounded-lg")} + className={cn(primaryControl, compactMobile ? "h-11 rounded-lg px-3 sm:px-5" : "h-11 rounded-lg")} > {loading ? : } Ask @@ -631,7 +633,7 @@ function MasterSearchHeader({
@@ -1398,39 +1400,47 @@ function UploadPanel({ }) { const fileRef = useRef(null); const [status, setStatus] = useState(""); + const [statusTone, setStatusTone] = useState<"neutral" | "success" | "warning" | "error">("neutral"); const [uploading, setUploading] = useState(false); async function submit(event: FormEvent) { event.preventDefault(); if (demoMode) { + setStatusTone("warning"); setStatus( "Demo mode is serving seeded documents. Configure .env.local, run supabase/schema.sql, and start npm run worker to upload real files.", ); return; } if (!canUpload) { + setStatusTone("warning"); setStatus("Sign in before uploading private guideline files."); return; } const file = fileRef.current?.files?.[0]; if (!file) { + setStatusTone("warning"); setStatus("Choose a PDF, DOCX, XLSX, or TXT file first."); return; } setUploading(true); + setStatusTone("neutral"); setStatus("Uploading private document to Supabase Storage..."); - const formData = new FormData(event.currentTarget); + const form = event.currentTarget; + const formData = new FormData(form); try { const response = await fetch("/api/upload", { method: "POST", headers: authorizationHeader, body: formData }); const payload = await response.json(); if (!response.ok) throw new Error(payload.error || "Upload failed"); - setStatus("Queued for local worker ingestion."); - event.currentTarget.reset(); + setStatusTone(payload.duplicate ? "warning" : "success"); + setStatus(payload.message ?? "Queued for local worker ingestion."); + form.reset(); onUploaded(); } catch (error) { + setStatusTone("error"); setStatus(error instanceof Error ? error.message : "Upload failed"); } finally { setUploading(false); @@ -1468,9 +1478,16 @@ function UploadPanel({ {(status || demoMode) && (

{status || @@ -1483,6 +1500,98 @@ function UploadPanel({ ); } +function formatBytes(bytes: number) { + if (!Number.isFinite(bytes) || bytes <= 0) return "0 MB"; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + +function IndexingMonitor({ + jobs, + batches, + actionId, + onRetry, + onReindex, +}: { + jobs: IngestionJob[]; + batches: ImportBatch[]; + actionId: string | null; + onRetry: (jobId: string) => void; + onReindex: (documentId: string) => void; +}) { + if (jobs.length === 0 && batches.length === 0) { + return ; + } + + return ( +

+ {batches.slice(0, 3).map((batch) => ( +
+
+
+

{batch.name}

+

+ {batch.total_files} files · {formatBytes(batch.total_bytes)} · {batch.queued_files} queued ·{" "} + {batch.skipped_files} exact copies skipped · {batch.failed_files} failed +

+
+ +
+
+ ))} + +

+ Keep `npm run worker` open while jobs are pending or processing. Failed jobs can be retried after fixing the + cause. +

+ + {jobs.slice(0, 10).map((job) => { + const documentTitle = job.documents?.title ?? job.documents?.file_name ?? "Document"; + const busy = actionId === job.id || actionId === job.document_id; + return ( +
+
+
+

{documentTitle}

+

{job.stage}

+
+ +
+
+
+
+
+ + Attempt {job.attempt_count ?? 0}/{job.max_attempts ?? 3} + + {job.status === "failed" && ( + + )} + +
+ {job.error_message &&

{job.error_message}

} +
+ ); + })} +
+ ); +} + const fallbackSetupChecks: SetupCheck[] = [ { id: "env", @@ -1733,6 +1842,7 @@ export function ClinicalDashboard() { const navSyncLockRef = useRef(null); const [documents, setDocuments] = useState([]); const [jobs, setJobs] = useState([]); + const [batches, setBatches] = useState([]); const [query, setQuery] = useState(""); const [answer, setAnswer] = useState(null); const [sources, setSources] = useState([]); @@ -1747,6 +1857,7 @@ export function ClinicalDashboard() { const [copiedAction, setCopiedAction] = useState(null); const [activeHash, setActiveHash] = useState("#search"); const [guideOpen, setGuideOpen] = useState(false); + const [indexingActionId, setIndexingActionId] = useState(null); const { theme, toggleTheme } = useTheme(); const auth = useAuthSession(); const { status: authStatus, authorizationHeader, markSessionExpired } = auth; @@ -1776,20 +1887,23 @@ export function ClinicalDashboard() { if (!nextDemoMode && authStatus !== "authenticated") { setDocuments([]); setJobs([]); + setBatches([]); return; } } const protectedHeaders = nextDemoMode ? undefined : authorizationHeader; - const [documentsResponse, jobsResponse] = await Promise.all([ + const [documentsResponse, jobsResponse, batchesResponse] = await Promise.all([ fetch("/api/documents", { headers: protectedHeaders }), - fetch("/api/jobs", { headers: protectedHeaders }), + fetch("/api/ingestion/jobs", { headers: protectedHeaders }), + fetch("/api/ingestion/batches", { headers: protectedHeaders }), ]); - if (documentsResponse.status === 401 || jobsResponse.status === 401) { + if (documentsResponse.status === 401 || jobsResponse.status === 401 || batchesResponse.status === 401) { markSessionExpired(); setDocuments([]); setJobs([]); + setBatches([]); return; } @@ -1810,8 +1924,56 @@ export function ClinicalDashboard() { } else { setApiUnavailable(true); } + + if (batchesResponse.ok) { + const payload = await batchesResponse.json(); + setBatches(payload.batches ?? []); + if (payload.demoMode) setDemoMode(true); + } else { + setApiUnavailable(true); + } }, [authStatus, authorizationHeader, clientDemoMode, markSessionExpired]); + const retryJob = useCallback( + async (jobId: string) => { + setIndexingActionId(jobId); + try { + const response = await fetch(`/api/ingestion/jobs/${jobId}/retry`, { + method: "POST", + headers: authorizationHeader, + }); + if (response.status === 401) { + markSessionExpired(); + return; + } + await refresh(); + } finally { + setIndexingActionId(null); + } + }, + [authorizationHeader, markSessionExpired, refresh], + ); + + const reindexDocument = useCallback( + async (documentId: string) => { + setIndexingActionId(documentId); + try { + const response = await fetch(`/api/documents/${documentId}/reindex`, { + method: "POST", + headers: authorizationHeader, + }); + if (response.status === 401) { + markSessionExpired(); + return; + } + await refresh(); + } finally { + setIndexingActionId(null); + } + }, + [authorizationHeader, markSessionExpired, refresh], + ); + useEffect(() => { const tick = () => { if (shouldPollForUpdates(demoMode, document.visibilityState)) { @@ -2315,28 +2477,13 @@ export function ClinicalDashboard() {

Indexing progress

- {jobs.length === 0 ? ( - - ) : ( - jobs.slice(0, 6).map((job) => ( -
-
-

{job.stage}

- -
-
-
-
-
- )) - )} +
@@ -2346,7 +2493,10 @@ export function ClinicalDashboard() {
-