Fix clinical dose-verification & safety gaps missed by early-merged #38

src/app/api/ingestion/jobs/[id]/retry/route.ts

@@ -24,35 +24,15 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
     if (jobError) throw new Error(jobError.message);
     if (!job) return NextResponse.json({ error: "Ingestion job not found." }, { status: 404 });
 
-    // IDX-C3: refuse to retry a job a live worker still holds. The claim RPC uses
-    // SKIP LOCKED + stale recovery; resetting here while status='processing' with a fresh
-    // lock would make the row claimable by a second worker, so two runs would insert against
-    // the same document_id concurrently and interleave mixed-generation clinical chunks.
-    if (job.status === "processing" && job.locked_at) {
-      const lockedAtMs = new Date(job.locked_at).getTime();
-      const staleAfterMs = env.WORKER_STALE_AFTER_MINUTES * 60_000;
-      const lockIsFresh = Number.isFinite(lockedAtMs) && Date.now() - lockedAtMs < staleAfterMs;
-      if (lockIsFresh) {
-        return NextResponse.json(
-          {
-            error:
-              "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.",
-          },
-          { status: 409 },
-        );
-      }
-    }
-
-    // IDX-H1: do NOT reset the document index here. The worker calls resetDocumentIndex at
-    // job start (worker/main.ts), so resetting before enqueue would leave a previously-good
-    // clinical document with zero index if the worker never runs or fails permanently. We
-    // only re-queue; the prior index stays live until the worker commits a fresh one.
-    const { error: documentError } = await supabase
-      .from("documents")
-      .update({ status: "queued", error_message: null })
-      .eq("id", job.document_id)
-      .eq("owner_id", user.id);
-    if (documentError) throw new Error(documentError.message);
+    // IDX-C3 / B6: refuse to retry a job a live worker still holds, atomically.
+    // A SELECT-then-UPDATE was a TOCTOU race: a worker could claim the job
+    // between the read and the write, and the unguarded UPDATE would silently
+    // reset the row the worker is processing → two workers ingest the same
+    // document_id and interleave mixed-generation clinical chunks. We instead
+    // make the reset a single conditional UPDATE whose WHERE clause refuses to
+    // touch a row that is freshly 'processing'. The reset only applies when the
+    // job is NOT processing, OR its lock is already stale, OR it has no lock.
+    const staleThreshold = new Date(Date.now() - env.WORKER_STALE_AFTER_MINUTES * 60_000).toISOString();
 
     const { data, error } = await supabase
       .from("ingestion_jobs")
@@ -69,10 +49,34 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
         completed_at: null,
       })
       .eq("id", id)
+      .or(`status.neq.processing,locked_at.is.null,locked_at.lt.${staleThreshold}`)
       .select()
-      .single();
+      .maybeSingle();
 
     if (error) throw new Error(error.message);
+    // 0 rows affected means the guard rejected the reset: the job is actively
+    // being processed with a fresh lock. Refuse rather than clobber it.
+    if (!data) {
+      return NextResponse.json(
+        {
+          error:
+            "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.",
+        },
+        { status: 409 },
+      );
+    }
+
+    // IDX-H1: do NOT reset the document index here. The worker calls resetDocumentIndex at
+    // job start (worker/main.ts), so resetting before enqueue would leave a previously-good
+    // clinical document with zero index if the worker never runs or fails permanently. We
+    // only re-queue; the prior index stays live until the worker commits a fresh one.
+    const { error: documentError } = await supabase
+      .from("documents")
+      .update({ status: "queued", error_message: null })
+      .eq("id", job.document_id)
+      .eq("owner_id", user.id);
+    if (documentError) throw new Error(documentError.message);
+
     return NextResponse.json({ job: data });
   } catch (error) {
     if (error instanceof AuthenticationError) return unauthorizedResponse();

src/lib/answer-verification.ts

@@ -18,54 +18,62 @@ import type { Citation, DocumentTableFact, SearchResult } from "@/lib/types";
 // and unit-bearing figures.
 // Hardened to include complex clinical units (mg/kg/day, mmol/L, etc.) and
 // ensure boundary safety to prevent hallucinated decimal bypass.
+// The ×10^n / ×10⁹ scientific-notation branch is listed BEFORE the bare unit
+// branches so "2.0 ×10⁹/L" is captured whole rather than truncated to "2.0".
+// The percentage branch must NOT be followed by \b: "%" is a non-word char, so
+// a trailing \b after it can never match (it would require a word char to its
+// right), which previously dropped every percentage token.
 const NUMERIC_TOKEN_PATTERN =
-  /\b\d+(?:[.,]\d+)?(?:\s*[-–—]\s*\d+(?:[.,]\d+)?)?\s*(?:%|mg\/(?:day|kg|m2|dose)|mg|mcg|micrograms?|μg|g\b|kg|ml|mL|l\b|mmol\/[lL]|mmol|mol\/[lL]|umol\/[lL]|µmol\/[lL]|ng\/ml|units?\/?\w*|iu\b|×10\^?\d*\/?l?|x10\^?\d*\/?l?|×10⁹\/l|hours?|hrs?|h\b|days?|weeks?|wk\b|months?|minutes?|mins?|years?|°c|mmhg|bpm)\b/giu;
+  /\b\d+(?:[.,]\d+)?(?:\s*[-–—]\s*\d+(?:[.,]\d+)?)?\s*(?:×10\^?\d*\/?l?|x10\^?\d*\/?l?|mg\/(?:day|kg|m2|dose)|mg|mcg|micrograms?|μg|g\b|kg|ml|mL|l\b|mmol\/[lL]|mmol|mol\/[lL]|umol\/[lL]|µmol\/[lL]|ng\/ml|units?\/?\w*|iu\b|hours?|hrs?|h\b|days?|weeks?|wk\b|months?|minutes?|mins?|years?|°c|mmhg|bpm)\b|\b\d+(?:[.,]\d+)?\s*%/giu;
 
 // Decimal numbers and ranges that, while not unit-bearing, are very likely
 // clinical thresholds in context (e.g. "ANC 2.0", "INR 2-3"). We only treat a
 // bare number as significant when it is a decimal or part of a range, to avoid
 // flagging incidental integers.
 const SIGNIFICANT_BARE_NUMBER_PATTERN = /\b\d+(?:[.,]\d+)?\s*[-–—]\s*\d+(?:[.,]\d+)?\b|\b\d+\.\d+\b/gu;
 
+// Unicode superscript digits ⁰¹²³⁴⁵⁶⁷⁸⁹ → ASCII. Sources and answers mix
+// "×10⁹/L" with "x10^9/L"; folding superscripts to ASCII before extraction and
+// matching keeps ANC/WBC thresholds (central to the clozapine use-case) from
+// being mangled or silently dropped.
+const SUPERSCRIPT_DIGITS: Record<string, string> = {
+  "⁰": "0",
+  "¹": "1",
+  "²": "2",
+  "³": "3",
+  "⁴": "4",
+  "⁵": "5",
+  "⁶": "6",
+  "⁷": "7",
+  "⁸": "8",
+  "⁹": "9",
+};
+
+function foldSuperscripts(text: string): string {
+  return text.replace(/[⁰¹²³⁴⁵⁶⁷⁸⁹]/g, (ch) => SUPERSCRIPT_DIGITS[ch] ?? ch);
+}
+
 function normalizeNumericToken(raw: string): string {
-  return raw
+  return foldSuperscripts(raw)
     .toLowerCase()
     .replace(/\s+/g, "")
     .replace(/[–—]/g, "-")
     .replace(/,/g, ".")
     .replace(/µ/g, "μ")
+    .replace(/×10\^?(\d+)/g, "x10^$1")
+    .replace(/x10(\d)/g, "x10^$1")
     .trim();
 }
 
-// Build a normalized haystack from a single text blob so token lookups are
-// whitespace/case/dash insensitive (matches how normalizeNumericToken folds).
-function normalizeHaystack(text: string): string {
-  return text
-    .toLowerCase()
-    .replace(/[–—]/g, "-")
-    .replace(/,/g, ".")
-    .replace(/µ/g, "μ")
-    .replace(/\s+/g, " ");
-}
-
-// A token "appears" in the source if either the spaced or the de-spaced form is
-// present, so "2.0 mg" in the answer matches "2.0mg" or "2.0 mg" in the source.
-function haystackContainsToken(haystack: string, despacedHaystack: string, token: string): boolean {
-  const normalized = token; // already normalized/de-spaced by caller
-  if (despacedHaystack.includes(normalized)) return true;
-  // Also try a spaced variant: re-insert a space between number and unit.
-  const spaced = normalized.replace(/^(\d+(?:\.\d+)?(?:-\d+(?:\.\d+)?)?)/, "$1 ");
-  return haystack.includes(spaced);
-}
-
 export function extractNumericTokens(text: string): string[] {
   if (!text) return [];
+  const folded = foldSuperscripts(text);
   const tokens = new Set<string>();
-  for (const match of text.matchAll(NUMERIC_TOKEN_PATTERN)) {
+  for (const match of folded.matchAll(NUMERIC_TOKEN_PATTERN)) {
     const normalized = normalizeNumericToken(match[0]);
     if (normalized) tokens.add(normalized);
   }
-  for (const match of text.matchAll(SIGNIFICANT_BARE_NUMBER_PATTERN)) {
+  for (const match of folded.matchAll(SIGNIFICANT_BARE_NUMBER_PATTERN)) {
     const normalized = normalizeNumericToken(match[0]);
     if (normalized) tokens.add(normalized);
   }
@@ -97,9 +105,15 @@ export type NumericVerification = {
   hasUnverifiedNumbers: boolean;
 };
 
-// Verify every numeric token in `answerText` against the combined text of the
+// Verify every numeric token in `answerText` against the numeric tokens of the
 // chunks the answer actually cites. Only cited chunks count — an answer that
 // cites nothing has no support, so all of its numbers are unverified.
+//
+// B1: matching is by SET MEMBERSHIP of normalized numeric tokens, never by
+// substring. Substring matching let "2.5 mg" match inside "12.5 mg" (a 5x dose
+// error) and "500 mg" inside "1500 mg". We extract the source's tokens with the
+// SAME extractor used on the answer, so a number is verified only when an exact
+// normalized token (e.g. "2.5mg") is present in the source's token set.
 export function verifyAnswerNumbers(
   answerText: string,
   citations: Array<Pick<Citation, "chunk_id">>,
@@ -112,17 +126,16 @@ export function verifyAnswerNumbers(
 
   const citedIds = new Set(citations.map((citation) => citation.chunk_id));
   const citedResults = results.filter((result) => citedIds.has(result.id));
-  // If no citation maps to a known chunk, fall back to all results so we don't
-  // over-flag during transitional states; cited-only is preferred when present.
-  const sourceResults = citedResults.length > 0 ? citedResults : results;
-
-  const combined = sourceResults.map(sourceTextForResult).join(" \n ");
-  const haystack = normalizeHaystack(combined);
-  const despacedHaystack = haystack.replace(/\s+/g, "");
+  // N1: fail closed. If nothing the answer cites maps to a known chunk, we have
+  // no scoped evidence — treat every numeric token as unverified rather than
+  // matching against the full unfiltered result set (which could "verify" a
+  // number that lives in a chunk the answer never cited).
+  if (citedResults.length === 0) {
+    return { answerTokens, unverifiedTokens: answerTokens, hasUnverifiedNumbers: answerTokens.length > 0 };
+  }
 
-  const unverifiedTokens = answerTokens.filter(
-    (token) => !haystackContainsToken(haystack, despacedHaystack, token),
-  );
+  const sourceTokens = sourceNumericTokenSet(citedResults);
+  const unverifiedTokens = answerTokens.filter((token) => !sourceTokens.has(token));
 
   return {
     answerTokens,
@@ -131,5 +144,18 @@ export function verifyAnswerNumbers(
   };
 }
 
+// Build the union of normalized numeric tokens across a set of source chunks
+// using the same extractor applied to the answer, so verification is an exact
+// token-membership test rather than a substring scan.
+function sourceNumericTokenSet(results: SearchResult[]): Set<string> {
+  const tokens = new Set<string>();
+  for (const result of results) {
+    for (const token of extractNumericTokens(sourceTextForResult(result))) {
+      tokens.add(token);
+    }
+  }
+  return tokens;
+}
+
 export const VERIFY_AGAINST_SOURCE_NOTE =
   "CRITICAL: Some figures in this answer could not be matched verbatim to the cited sources — verify against the source documents before acting.";

src/lib/rag-routing.ts

@@ -335,10 +335,16 @@ export function chooseAnswerRoute(args: {
 
 export function shouldRetryWithStrongAfterFast(args: {
   route: AnswerRoute;
-  answer: Pick<RagAnswer, "grounded" | "confidence" | "citations">;
+  answer: Pick<RagAnswer, "grounded" | "confidence" | "citations" | "routingReason">;
   results: SearchResult[];
 }) {
   if (args.route.mode !== "fast") return false;
+  // A structured-parse failure is a generation/format problem, not a weak-
+  // retrieval signal. Since B5 made that fallback fail closed (ungrounded, no
+  // citations), retrying with the strong model would re-run generation (often
+  // re-truncating) instead of letting extractive recovery use the retrieved
+  // sources. Skip the strong retry here; extractive recovery handles it.
+  if (args.answer.routingReason === "structured_parse_fallback") return false;
   if (args.answer.grounded && args.answer.confidence !== "unsupported" && args.answer.citations.length > 0) {
     return false;
   }

src/lib/rag.ts

@@ -617,20 +617,25 @@ function normalizeSearchResults(results: SearchResult[]) {
 }
 
 function safeFallbackAnswer(raw: string, results: SearchResult[], query?: string): RagAnswer {
-  const citations = compactCitations(results);
-  const confidence = deriveConfidence(results, citations.length);
-  return {
+  // B5: on model-JSON parse failure we cannot trust any model-asserted citation
+  // mapping. Do NOT back-fill all retrieved chunks as citations and stamp the
+  // answer grounded — that re-introduces exactly the back-fill GEN-C3 removed,
+  // hidden in the error path. Treat a parse failure as ungrounded/unsupported,
+  // and still run the numeric faithfulness gate over the salvaged prose so any
+  // dose/threshold it contains is surfaced as unverified rather than trusted.
+  const answer: RagAnswer = {
     answer: boldHighYieldClinicalText(sanitizeAnswerText(raw) || machineReadableFallbackAnswer, query),
-    grounded: citations.length > 0 && confidence !== "unsupported",
-    confidence,
-    citations,
+    grounded: false,
+    confidence: "unsupported",
+    citations: [],
     sources: results,
     routingReason: "structured_parse_fallback",
     answerSections: [],
     conflictsOrGaps: detectConflictsOrGaps(results),
     visualEvidence: buildVisualEvidence(results),
     bestSource: selectBestSourceRecommendation(results),
   };
+  return applyNumericVerification(answer);
 }
 
 function addOpenAIUsage(total: OpenAITokenUsage, usage?: OpenAITokenUsage) {
@@ -3395,7 +3400,7 @@ export function parseAnswerJson(raw: string, results: SearchResult[], query?: st
       answer.routingReason = "ungrounded_no_model_citation";
     }
     // GEN-C2 / GEN-H2: numeric faithfulness gate.
-    return applyNumericVerification(answer, query);
+    return applyNumericVerification(answer);
   } catch {
     return safeFallbackAnswer(raw, results, query);
   }
@@ -3405,21 +3410,41 @@ export function parseAnswerJson(raw: string, results: SearchResult[], query?: st
 // answer against the text of its cited chunks. Unsupported figures are recorded
 // on the answer and an explicit "verify against source" caveat is appended so a
 // paraphrased/mis-transcribed dose can never read as authoritative.
-function applyNumericVerification(answer: RagAnswer, query?: string): RagAnswer {
-  const verification = verifyAnswerNumbers(answer.answer, answer.citations, answer.sources ?? []);
-  if (!verification.hasUnverifiedNumbers) return answer;
+function applyNumericVerification(answer: RagAnswer): RagAnswer {
+  const sources = answer.sources ?? [];
+  const unverified = new Set<string>();
+
+  // B4: the model is instructed to put dose details in structured
+  // answerSections (kind medication_dose), so a top-level-only scan never sees
+  // section-body doses. Verify the top-level answer AND every section body.
+  // Each section is scoped to its own citation_chunk_ids when present, so a
+  // dose is only credited against the chunks that section actually cites;
+  // sections with no citations fall back to the answer-level citations.
+  const answerVerification = verifyAnswerNumbers(answer.answer, answer.citations, sources);
+  for (const token of answerVerification.unverifiedTokens) unverified.add(token);
+
+  for (const section of answer.answerSections ?? []) {
+    const sectionCitations =
+      section.citation_chunk_ids.length > 0
+        ? section.citation_chunk_ids.map((chunk_id) => ({ chunk_id }))
+        : answer.citations;
+    const sectionVerification = verifyAnswerNumbers(section.body, sectionCitations, sources);
+    for (const token of sectionVerification.unverifiedTokens) unverified.add(token);
+  }
+
+  if (unverified.size === 0) return answer;
 
-  answer.unverifiedNumericTokens = verification.unverifiedTokens;
+  const unverifiedTokens = [...unverified];
+  answer.unverifiedNumericTokens = unverifiedTokens;
   answer.faithfulnessWarning = VERIFY_AGAINST_SOURCE_NOTE;
   // Surface as a source gap so the UI's existing gap rendering shows it, and
   // never let an answer with unverified clinical numbers claim high confidence.
   const caveat: ConflictOrGap = {
     type: "gap",
-    message: `${VERIFY_AGAINST_SOURCE_NOTE} Unverified figures: ${verification.unverifiedTokens.join(", ")}.`,
+    message: `${VERIFY_AGAINST_SOURCE_NOTE} Unverified figures: ${unverifiedTokens.join(", ")}.`,
   };
   answer.conflictsOrGaps = [...(answer.conflictsOrGaps ?? []), caveat];
   if (answer.confidence === "high") answer.confidence = "medium";
-  void query;
   return answer;
 }
 
@@ -4015,7 +4040,13 @@ ${buildRagSourceBlock(contextResults, { query: answerFocusQuery, queryClass })}`
       total_latency_ms: Date.now() - startedAt,
     };
 
-    if (answer.citations.length > 0 && isUnusableGeneratedAnswer(answer)) {
+    // B5: a structured_parse_fallback answer now fails closed with zero
+    // citations, so we can no longer gate extractive recovery on the parsed
+    // answer's citations. buildExtractiveAnswer derives its own source-backed
+    // citations from the retrieved results, so trigger recovery whenever the
+    // generated answer is unusable and we have retrieved results to extract from.
+    const canRecoverExtractively = answer.citations.length > 0 || answerInputResults.length > 0;
+    if (canRecoverExtractively && isUnusableGeneratedAnswer(answer)) {
       answer = buildExtractiveAnswer({
         query: args.query,
         queryClass,