Improve Clinical KB design and RAG hardening

.env.example

@@ -7,14 +7,25 @@ SUPABASE_PROJECT_NAME=Clinical KB Database
 NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=your-publishable-or-anon-key
 SUPABASE_SERVICE_ROLE_KEY=your-service-role-key
 
+# Local-only no-auth mode (development only).
+# Enable one or both of these to load real data without per-request browser auth:
+# - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server)
+# - LOCAL_NO_AUTH (server only)
+#NEXT_PUBLIC_LOCAL_NO_AUTH=false
+#LOCAL_NO_AUTH=false
+
+# Optional local no-auth owner identity. Set one of these in local dev.
+#LOCAL_NO_AUTH_OWNER_ID=
+#LOCAL_NO_AUTH_OWNER_EMAIL=
+
 # OpenAI direct API. This app sends extracted guideline text and extracted images
 # to OpenAI for embeddings, captioning, and grounded answer generation.
 OPENAI_API_KEY=replace-with-openai-api-key
 OPENAI_EMBEDDING_MODEL=text-embedding-3-small
 OPENAI_ANSWER_MODEL=gpt-5.4-mini
 OPENAI_FAST_ANSWER_MODEL=gpt-5.4-mini
 OPENAI_STRONG_ANSWER_MODEL=gpt-5.4
-OPENAI_MAX_OUTPUT_TOKENS=900
+OPENAI_MAX_OUTPUT_TOKENS=1400
 OPENAI_QUERY_CACHE_SIZE=200
 OPENAI_VISION_MODEL=gpt-5.4-mini
 OPENAI_REQUEST_TIMEOUT_MS=45000

.github/workflows/ci.yml

@@ -13,10 +13,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: 22
           cache: npm

.github/workflows/secret-scan.yml

@@ -19,11 +19,11 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Scan for secrets
-        uses: gitleaks/gitleaks-action@v2
+        uses: gitleaks/gitleaks-action@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

AGENTS.md

@@ -27,3 +27,103 @@ This version has breaking changes — APIs, conventions, and file structure may
 - Older unused project ref `qjgitjyhxrwxsrydablr` belongs to `Database`; treat it as stale and do not use it.
 - Run `npm run check:supabase-project` after changing Supabase env values.
 <!-- END:supabase-project-safety -->
+
+<!-- BEGIN:upload-shortcut -->
+
+# `upload` shortcut
+
+When the user types exactly:
+
+upload
+
+as the entire task message, treat it as a shortcut for the safe Git handoff workflow below.
+
+The goal is to leave useful completed work safely committed and, where safe, pushed to the current feature branch. The goal is not to merge into main, delete branches, discard work, force-push, close PRs, deploy, or perform destructive cleanup without explicit user confirmation.
+
+## Protected and base branches
+
+Treat `main`, `master`, `develop`, and `release/*` as protected/base branches for this workflow.
+
+If `upload` is run while on `main`, automatically create or use a branch named exactly `temporary` before staging, committing, or pushing, then continue the upload workflow from `temporary`:
+
+- If neither local `temporary` nor `origin/temporary` exists, run `git switch -c temporary`.
+- If local `temporary` exists and is not checked out in another worktree, switch to it only when it is clearly safe.
+- If `origin/temporary` exists, use it only when it is clearly the matching intended branch.
+- If any `temporary` branch state is ambiguous, diverged, checked out elsewhere, or unsafe, stop and ask instead of overwriting.
+
+If already on a non-protected feature branch, continue using that branch.
+
+## Required inspection
+
+Start with read-only inspection before making changes. Check:
+
+- Current branch or detached HEAD state
+- `git status`
+- Staged, unstaged, and untracked files
+- Recent commits relevant to the current branch
+- Remote configuration and upstream branch
+- Whether the branch is ahead, behind, or diverged
+- Whether the current branch appears protected/base
+- Other Git worktrees, if detectable
+- Available checks such as tests, lint, type check, or build scripts
+- Existing branch, commit, PR, and release-flow conventions
+
+Do not assume branch names, remotes, package managers, test commands, deployment targets, or project structure. Inspect first.
+
+## Safe actions allowed without further confirmation
+
+When the repository state makes it clearly safe, you may:
+
+- Stage coherent completed changes that clearly belong together
+- Create one or more logical commits with clear messages based on the diff
+- Fast-forward pull only when there are no local commits or conflict risks
+- Push the current non-protected feature branch if it has a valid upstream
+- Set an upstream for the current feature branch only when the correct remote and branch name are obvious
+- Leave the worktree clean by committing safe completed changes
+
+## Actions requiring explicit confirmation
+
+Do not perform these without asking the user first:
+
+- `git reset --hard`
+- `git clean -fd` or other destructive cleanup
+- Discarding, overwriting, or reverting uncommitted changes
+- Deleting local or remote branches
+- Renaming branches
+- Force-pushing
+- Rebasing a shared/public branch
+- Resolving divergent branch history
+- Merging into `main`, `master`, `develop`, `release/*`, or any protected/base branch
+- Closing pull requests
+- Changing GitHub default branch, branch protection, repository settings, or deployment settings
+- Modifying production data or deployment configuration
+- Committing secrets, credentials, tokens, private keys, or sensitive local configuration
+- Updating branch references where the correct replacement branch is ambiguous
+
+If any of these seem necessary, stop and report what is risky, why it is risky, the recommended next step, and the exact confirmation needed.
+
+## Mixed, suspicious, or unsafe changes
+
+Do not automatically commit files that look like `.env` files, credentials, secrets, logs, caches, build artifacts, editor or OS files, temporary/debug files, or generated files not normally committed by this project. Report only the path and concern for possible secrets; never print secret values.
+
+If changes appear unrelated, incomplete, experimental, or WIP, do not commit everything together automatically. Commit only clearly coherent completed changes when safe; otherwise summarize the groups and ask what should be included.
+
+## Branch cleanup and reference updates
+
+If stale, inappropriate, merged, or unnecessary branches are detected, list cleanup candidates but do not delete or rename branches automatically.
+
+Before recommending deletion or rename, audit accessible references including `.github/workflows/*`, CI/CD config, deployment config, scripts, package scripts, docs, release notes or release scripts, safe environment/config files, branch-specific config, open PR metadata if accessible, and GitHub branch protection/default branch metadata if safely accessible.
+
+Update repo-tracked references to a renamed or replacement branch only when the old branch reference is clearly found, the replacement is obvious, the change is low-risk, and the user has approved the branch rename or deletion. If the replacement is unclear, report the reference and ask what it should point to.
+
+## Syncing and verification
+
+Do not rebase, merge, or resolve remote divergence automatically. Fast-forward pulls are allowed only when clearly safe. Push only the current non-protected feature branch when clearly safe.
+
+Run the smallest relevant checks that are available and appropriate, such as tests, lint, type check, or build checks. Do not claim checks passed unless they were actually run. If checks cannot be run, explain why and state the command that would normally be used.
+
+## Final report
+
+After completing `upload`, summarize the current branch and worktree state, whether the worktree is clean, what changed, files committed, commit hash and message if created, whether anything was pushed, remote branch and likely PR target, checks run and results, checks not run and why, branch cleanup candidates, branch references found or updated, risky actions skipped, and exact confirmation needed for any recommended follow-up.
+
+<!-- END:upload-shortcut -->

package.json

@@ -24,6 +24,9 @@
     "check:indexing": "tsx scripts/check-indexing.ts",
     "eval:rag": "tsx scripts/eval-rag.ts",
     "eval:search": "tsx scripts/eval-search.ts",
+    "cleanup:storage": "tsx scripts/cleanup-storage.ts",
+    "purge:query-logs": "tsx scripts/purge-query-logs.ts",
+    "audit:tables": "tsx scripts/audit-tables.ts",
     "samples": "tsx scripts/generate-sample-documents.ts",
     "samples:check": "tsx scripts/check-sample-extraction.ts"
   },

scripts/audit-tables.ts

@@ -0,0 +1,188 @@
+import { loadEnvConfig } from "@next/env";
+import { findOwnerIdByEmail, loadAdminClient } from "./eval-utils";
+import { assessClinicalImageUse } from "@/lib/image-filtering";
+
+loadEnvConfig(process.cwd());
+
+type Args = {
+  ownerEmail?: string;
+  allOwners: boolean;
+  document?: string;
+  limit: number;
+  failOnMissed: boolean;
+};
+
+function parseArgs(): Args {
+  const args: Args = { ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, allOwners: !process.env.RAG_EVAL_OWNER_EMAIL, limit: 200, failOnMissed: false };
+  const tokens = process.argv.slice(2);
+  for (let index = 0; index < tokens.length; index += 1) {
+    const token = tokens[index];
+    const value = tokens[index + 1];
+    if (token === "--owner-email") {
+      args.ownerEmail = value;
+      args.allOwners = false;
+      index += 1;
+    } else if (token === "--all-owners") {
+      args.allOwners = true;
+    } else if (token === "--document") {
+      args.document = value;
+      index += 1;
+    } else if (token === "--limit") {
+      args.limit = Math.max(1, Math.min(Number(value) || 200, 1000));
+      index += 1;
+    } else if (token === "--fail-on-missed") {
+      args.failOnMissed = true;
+    }
+  }
+  return args;
+}
+
+function textSuggestsTable(text: string) {
+  return /\b(table\s+\d+[a-z]?|appendix\s+\d+[a-z]?|roles?\s+and\s+responsibilities|score\b.*\bmanagement|observation|medication|dose|frequency)\b/i.test(
+    text,
+  );
+}
+
+function compactTitle(title: string) {
+  return title.length > 72 ? `${title.slice(0, 69).trim()}...` : title;
+}
+
+async function main() {
+  const args = parseArgs();
+  if (!args.ownerEmail && !args.allOwners) throw new Error('Provide --owner-email "you@example.com" or --all-owners.');
+
+  const supabase = await loadAdminClient();
+  const ownerId = args.ownerEmail && !args.allOwners ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined;
+
+  let documentQuery = supabase
+    .from("documents")
+    .select("id,title,file_name,status,page_count,image_count")
+    .order("created_at", { ascending: true })
+    .limit(args.limit);
+  if (ownerId) documentQuery = documentQuery.eq("owner_id", ownerId);
+  if (args.document) {
+    documentQuery = documentQuery.or(
+      `id.eq.${args.document},file_name.ilike.%${args.document}%,title.ilike.%${args.document}%`,
+    );
+  }
+
+  const { data: documents, error: documentsError } = await documentQuery;
+  if (documentsError) throw new Error(documentsError.message);
+  const documentIds = (documents ?? []).map((document) => document.id as string);
+  if (documentIds.length === 0) {
+    console.log("No documents matched the audit filters.");
+    return;
+  }
+
+  const [imagesResult, pagesResult] = await Promise.all([
+    supabase
+      .from("document_images")
+      .select("document_id,source_kind,searchable,image_type,clinical_relevance_score,metadata")
+      .in("document_id", documentIds)
+      .neq("image_type", "logo_decorative"),
+    supabase.from("document_pages").select("document_id,page_number,text").in("document_id", documentIds),
+  ]);
+  if (imagesResult.error) throw new Error(imagesResult.error.message);
+  if (pagesResult.error) throw new Error(pagesResult.error.message);
+
+  const counts = new Map<string, { tables: number; clinicalTables: number; adminTables: number; searchableAdminTables: number; images: number }>();
+  for (const documentId of documentIds) {
+    counts.set(documentId, { tables: 0, clinicalTables: 0, adminTables: 0, searchableAdminTables: 0, images: 0 });
+  }
+  for (const image of imagesResult.data ?? []) {
+    const documentId = String(image.document_id);
+    const current = counts.get(documentId) ?? {
+      tables: 0,
+      clinicalTables: 0,
+      adminTables: 0,
+      searchableAdminTables: 0,
+      images: 0,
+    };
+    const metadata = image.metadata && typeof image.metadata === "object" ? (image.metadata as Record<string, unknown>) : {};
+    const useClass = String(
+      metadata.clinical_use_class ??
+        assessClinicalImageUse({
+          imageType: image.image_type,
+          searchable: image.searchable,
+          clinicalRelevanceScore: image.clinical_relevance_score,
+          sourceKind: image.source_kind,
+          tableRole: typeof metadata.table_role === "string" ? metadata.table_role : null,
+          tableText:
+            typeof metadata.table_text === "string"
+              ? metadata.table_text
+              : typeof metadata.table_text_snippet === "string"
+                ? metadata.table_text_snippet
+                : null,
+        }).clinical_use_class,
+    );
+    if (image.searchable !== false) current.images += 1;
+    if (image.source_kind === "table_crop") {
+      current.tables += 1;
+      if (useClass === "clinical_evidence" && image.searchable !== false) current.clinicalTables += 1;
+      if (["administrative", "reference"].includes(useClass)) current.adminTables += 1;
+      if (["administrative", "reference"].includes(useClass) && image.searchable !== false) {
+        current.searchableAdminTables += 1;
+      }
+    }
+    counts.set(documentId, current);
+  }
+
+  const tableMarkerPages = new Map<string, Set<number>>();
+  for (const page of pagesResult.data ?? []) {
+    if (!textSuggestsTable(String(page.text ?? ""))) continue;
+    const documentId = String(page.document_id);
+    const pages = tableMarkerPages.get(documentId) ?? new Set<number>();
+    const pageNumber = Number(page.page_number);
+    if (Number.isFinite(pageNumber)) pages.add(pageNumber);
+    tableMarkerPages.set(documentId, pages);
+  }
+
+  const possibleMisses: string[] = [];
+  const searchableAdminIssues: string[] = [];
+  console.log(`Table audit for ${documents?.length ?? 0} documents`);
+  for (const document of documents ?? []) {
+    const documentCounts = counts.get(document.id) ?? {
+      tables: 0,
+      clinicalTables: 0,
+      adminTables: 0,
+      searchableAdminTables: 0,
+      images: 0,
+    };
+    const markerPages = Array.from(tableMarkerPages.get(document.id) ?? []).sort((a, b) => a - b);
+    if (markerPages.length > 0 && documentCounts.tables === 0) {
+      possibleMisses.push(`${document.file_name}: table-like text on pages ${markerPages.slice(0, 8).join(", ")}`);
+    }
+    if (documentCounts.searchableAdminTables > 0) {
+      searchableAdminIssues.push(`${document.file_name}: searchable admin/reference tables=${documentCounts.searchableAdminTables}`);
+    }
+    console.log(
+      [
+        compactTitle(document.file_name ?? document.title ?? document.id),
+        `status=${document.status}`,
+        `pages=${document.page_count ?? 0}`,
+        `tables=${documentCounts.tables}`,
+        `clinicalTables=${documentCounts.clinicalTables}`,
+        `adminReferenceTables=${documentCounts.adminTables}`,
+        `searchableAdminReference=${documentCounts.searchableAdminTables}`,
+        `searchableImages=${documentCounts.images}`,
+        markerPages.length ? `tableTextPages=${markerPages.slice(0, 8).join(",")}` : "tableTextPages=none",
+      ].join(" | "),
+    );
+  }
+
+  console.log(`Possible missed table documents: ${possibleMisses.length}`);
+  for (const item of possibleMisses.slice(0, 20)) console.log(`- ${item}`);
+  console.log(`Searchable admin/reference table issues: ${searchableAdminIssues.length}`);
+  for (const item of searchableAdminIssues.slice(0, 20)) console.log(`- ${item}`);
+  if (searchableAdminIssues.length > 0) {
+    throw new Error("Table audit found admin/reference tables still marked searchable.");
+  }
+  if (args.failOnMissed && possibleMisses.length > 0) {
+    throw new Error("Table audit found documents with table-like text but no retained table crops.");
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : error);
+  process.exit(1);
+});