Fix all Critical & High RAG pipeline issues (indexing, retrieval, generation)

src/app/api/documents/[id]/reindex/route.ts

@@ -129,16 +129,11 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
       });
     }
 
-    const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: id });
-    if (resetError) throw new Error(resetError.message);
-
-    const { error: updateError } = await supabase
-      .from("documents")
-      .update({ status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 })
-      .eq("id", id)
-      .eq("owner_id", user.id);
-    if (updateError) throw new Error(updateError.message);
-
+    // IDX-H1: enqueue the job first, then mark queued. Do NOT reset the index here — the
+    // worker calls resetDocumentIndex at job start (worker/main.ts). Resetting before the
+    // job is committed would leave a previously-searchable clinical document with zero index
+    // if job creation failed or the worker never ran (silent availability regression). The
+    // existing index stays live until the worker commits a fresh one.
     const { data: job, error: jobError } = await supabase
       .from("ingestion_jobs")
       .insert({
@@ -153,6 +148,14 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
       .single();
 
     if (jobError) throw new Error(jobError.message);
+
+    const { error: updateError } = await supabase
+      .from("documents")
+      .update({ status: "queued", error_message: null })
+      .eq("id", id)
+      .eq("owner_id", user.id);
+    if (updateError) throw new Error(updateError.message);
+
     return NextResponse.json({ job }, { status: 201 });
   } catch (error) {
     if (error instanceof AuthenticationError) return unauthorizedResponse();

src/app/api/documents/bulk/reindex/route.ts

@@ -127,14 +127,9 @@ export async function POST(request: Request) {
           continue;
         }
 
-        const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: document.id });
-        if (resetError) throw new Error(resetError.message);
-        const { error: updateError } = await supabase
-          .from("documents")
-          .update({ status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 })
-          .eq("id", document.id)
-          .eq("owner_id", user.id);
-        if (updateError) throw new Error(updateError.message);
+        // IDX-H1: enqueue first, then mark queued. Do NOT reset the index here — the worker
+        // resets at job start (worker/main.ts). Resetting before the job is committed would
+        // leave a previously-searchable clinical document with zero index on failure.
         const { data: job, error: jobError } = await supabase
           .from("ingestion_jobs")
           .insert({
@@ -148,6 +143,12 @@ export async function POST(request: Request) {
           .select("id")
           .single();
         if (jobError) throw new Error(jobError.message);
+        const { error: updateError } = await supabase
+          .from("documents")
+          .update({ status: "queued", error_message: null })
+          .eq("id", document.id)
+          .eq("owner_id", user.id);
+        if (updateError) throw new Error(updateError.message);
         results.push({ documentId: document.id, mode: parsed.data.mode, ok: true, jobId: job.id });
       } catch (error) {
         results.push({

src/app/api/ingestion/jobs/[id]/retry/route.ts

@@ -16,20 +16,40 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
 
     const { data: job, error: jobError } = await supabase
       .from("ingestion_jobs")
-      .select("id,document_id,batch_id,documents!inner(owner_id)")
+      .select("id,document_id,batch_id,status,locked_at,documents!inner(owner_id)")
       .eq("id", id)
       .eq("documents.owner_id", user.id)
       .maybeSingle();
 
     if (jobError) throw new Error(jobError.message);
     if (!job) return NextResponse.json({ error: "Ingestion job not found." }, { status: 404 });
 
-    const { error: resetError } = await supabase.rpc("reset_document_index", { p_document_id: job.document_id });
-    if (resetError) throw new Error(resetError.message);
+    // IDX-C3: refuse to retry a job a live worker still holds. The claim RPC uses
+    // SKIP LOCKED + stale recovery; resetting here while status='processing' with a fresh
+    // lock would make the row claimable by a second worker, so two runs would insert against
+    // the same document_id concurrently and interleave mixed-generation clinical chunks.
+    if (job.status === "processing" && job.locked_at) {
+      const lockedAtMs = new Date(job.locked_at).getTime();
+      const staleAfterMs = env.WORKER_STALE_AFTER_MINUTES * 60_000;
+      const lockIsFresh = Number.isFinite(lockedAtMs) && Date.now() - lockedAtMs < staleAfterMs;
+      if (lockIsFresh) {
+        return NextResponse.json(
+          {
+            error:
+              "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.",
+          },
+          { status: 409 },
+        );
+      }
+    }
 
+    // IDX-H1: do NOT reset the document index here. The worker calls resetDocumentIndex at
+    // job start (worker/main.ts), so resetting before enqueue would leave a previously-good
+    // clinical document with zero index if the worker never runs or fails permanently. We
+    // only re-queue; the prior index stays live until the worker commits a fresh one.
     const { error: documentError } = await supabase
       .from("documents")
-      .update({ status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 })
+      .update({ status: "queued", error_message: null })
       .eq("id", job.document_id)
       .eq("owner_id", user.id);
     if (documentError) throw new Error(documentError.message);

src/app/api/search/interaction/route.ts

@@ -1,7 +1,10 @@
 import { NextResponse } from "next/server";
 import { z } from "zod";
 import { normalizedClinicalSearchTokens } from "@/lib/clinical-search";
+import { isDemoMode } from "@/lib/env";
+import { normalizeQueryText, queryPrivacyMetadata, queryTextForStorage } from "@/lib/query-privacy";
 import { createAdminClient } from "@/lib/supabase/admin";
+import * as serverAuth from "@/lib/supabase/auth";
 
 export const runtime = "nodejs";
 
@@ -17,36 +20,45 @@ const interactionSchema = z.object({
 export async function POST(request: Request) {
   try {
     const body = interactionSchema.parse(await request.json());
-    const normalizedQuery = body.query.toLowerCase().replace(/\s+/g, " ").trim();
-    await createAdminClient()
-      .from("rag_query_misses")
-      .insert({
-        owner_id: null,
-        query: body.query,
-        normalized_query: normalizedQuery,
-        query_class: body.queryClass ?? null,
-        clicked_document_id: body.documentId,
-        clicked_chunk_id: body.chunkId ?? null,
-        top_files: body.fileName ? [body.fileName] : [],
-        top_chunk_ids: body.chunkId ? [body.chunkId] : [],
-        miss_reason: "clicked_result",
-        candidate_aliases: normalizedClinicalSearchTokens(body.query).slice(0, 10),
-        candidate_labels: body.title
-          ? [
-              {
-                label: body.title,
-                label_type: "document_type",
-                document_id: body.documentId,
-                confidence: 0.6,
-              },
-            ]
-          : [],
-        metadata: {
-          interaction: "source_open",
-        },
-      });
+    if (isDemoMode()) {
+      return NextResponse.json({ ok: true });
+    }
+
+    const supabase = createAdminClient();
+    // Carry the authenticated owner through so the miss row is attributable and
+    // owner-cleanable instead of being orphaned with owner_id: null (RET-H4).
+    const user = await serverAuth.requireAuthenticatedUser(request, supabase);
+    await supabase.from("rag_query_misses").insert({
+      owner_id: user.id,
+      query: queryTextForStorage(body.query),
+      normalized_query: normalizeQueryText(body.query),
+      query_class: body.queryClass ?? null,
+      clicked_document_id: body.documentId,
+      clicked_chunk_id: body.chunkId ?? null,
+      top_files: body.fileName ? [body.fileName] : [],
+      top_chunk_ids: body.chunkId ? [body.chunkId] : [],
+      miss_reason: "clicked_result",
+      candidate_aliases: normalizedClinicalSearchTokens(body.query).slice(0, 10),
+      candidate_labels: body.title
+        ? [
+            {
+              label: body.title,
+              label_type: "document_type",
+              document_id: body.documentId,
+              confidence: 0.6,
+            },
+          ]
+        : [],
+      metadata: {
+        interaction: "source_open",
+        ...queryPrivacyMetadata(body.query),
+      },
+    });
     return NextResponse.json({ ok: true });
-  } catch {
+  } catch (error) {
+    if (error instanceof serverAuth.AuthenticationError) {
+      return serverAuth.unauthorizedResponse(error);
+    }
     return NextResponse.json({ ok: false }, { status: 400 });
   }
 }

src/app/api/search/route.ts

@@ -16,6 +16,7 @@ import { consumePublicSearchRateLimit } from "@/lib/public-rate-limit";
 import { clinicalQueryModeSchema, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode";
 import { resolveSearchScope, searchScopeFiltersSchema } from "@/lib/search-scope";
 import { sourceGovernanceWarnings } from "@/lib/source-governance";
+import { normalizeQueryText, queryPrivacyMetadata, queryTextForStorage } from "@/lib/query-privacy";
 import type { ChunkImage, ClinicalSourceMetadata, SearchResult } from "@/lib/types";
 
 export const runtime = "nodejs";
@@ -347,8 +348,8 @@ function logWeakSearch(args: {
     .from("rag_query_misses")
     .insert({
       owner_id: args.ownerId,
-      query: args.query,
-      normalized_query: args.query.toLowerCase().replace(/\s+/g, " ").trim(),
+      query: queryTextForStorage(args.query),
+      normalized_query: normalizeQueryText(args.query),
       query_class: args.queryClass,
       route: args.route ?? null,
       retrieval_strategy: args.retrievalStrategy ?? null,
@@ -365,6 +366,7 @@ function logWeakSearch(args: {
         relevance_score: args.relevance.score,
         direct_source_count: args.relevance.directSourceCount,
         weak_source_count: args.relevance.weakSourceCount,
+        ...queryPrivacyMetadata(args.query),
       },
     })
     .then(undefined, () => undefined);
@@ -404,11 +406,12 @@ function logSearchObservation(args: {
       const latencyMs = telemetryLatencyMs(telemetry);
       await args.supabase.from("rag_queries").insert({
         owner_id: args.ownerId,
-        query: args.query,
+        query: queryTextForStorage(args.query),
         answer: null,
         source_chunk_ids: args.results.map((result) => result.id),
         model: "search",
         metadata: {
+          ...queryPrivacyMetadata(args.query),
           event_type: args.failure ? "private_search_failure" : "private_search",
           failure_code: args.failure?.code ?? null,
           failure_cause_name: args.failure?.causeName ?? null,

src/components/AccessibleTable.tsx

@@ -3,7 +3,7 @@
 import { Maximize2, X } from "lucide-react";
 import { useCallback, useEffect, useId, useMemo, useRef, useState, useSyncExternalStore } from "react";
 import { cn, textMuted } from "@/components/ui-primitives";
-import { normalizeAccessibleTable } from "@/lib/accessible-table-normalization";
+import { normalizeAccessibleTable, type NormalizedAccessibleTable } from "@/lib/accessible-table-normalization";
 
 const tableExpandMediaQuery = "(max-width: 768px), ((max-width: 1023px) and (hover: none) and (pointer: coarse))";
 const metadataHeaderPattern = /^(?:source|sources|support|pages?|chunk|file|document|citation|citations|provenance)$/i;
@@ -42,7 +42,7 @@ function isMetadataHeader(value: string) {
   return metadataHeaderPattern.test(value.trim());
 }
 
-function clinicalOnlyTable(table: { header: string[]; body: string[][] }) {
+function clinicalOnlyTable(table: NormalizedAccessibleTable) {
   const keptIndexes = table.header
     .map((header, index) => ({ header, index }))
     .filter(({ header }) => !isMetadataHeader(header))
@@ -55,7 +55,9 @@ function clinicalOnlyTable(table: { header: string[]; body: string[][] }) {
     .filter((row) => row.some(Boolean));
 
   if (!header.length || !body.length) return null;
-  return { header, body };
+  // GEN-H3: carry the low-confidence flag through so an ambiguously-normalized
+  // clinical table is rendered with a "verify against source" caveat.
+  return { header, body, lowConfidence: table.lowConfidence, lowConfidenceReason: table.lowConfidenceReason };
 }
 
 function useMobileTableExpansion(enabled: boolean) {
@@ -210,7 +212,20 @@ export function AccessibleTable({
   const { header, body } = normalized;
   const displayCaption = clinicalOnly && caption ? cleanClinicalTableText(caption) : caption;
   const title = dialogTitle || displayCaption || "Clinical table";
-  const table = <AccessibleTableMarkup caption={displayCaption} header={header} body={body} compact={compact} />;
+  const lowConfidence = Boolean(normalized.lowConfidence);
+  const table = (
+    <>
+      {lowConfidence ? (
+        <p
+          data-testid="table-low-confidence-note"
+          className={cn("mb-1 text-xs", textMuted)}
+        >
+          Table structure could not be confidently reconstructed — verify values against the source document.
+        </p>
+      ) : null}
+      <AccessibleTableMarkup caption={displayCaption} header={header} body={body} compact={compact} />
+    </>
+  );
 
   function openDialog(trigger: HTMLElement) {
     if (!canExpand) return;