feat(rag): remediate clinical retrieval and safety

.env.example

@@ -107,6 +107,8 @@ SUPABASE_IMAGE_BUCKET=clinical-images
 MAX_UPLOAD_MB=150
 # Next Proxy has a fixed 151 MiB transport envelope (150 MiB file plus
 # multipart framing), so values above 150 are intentionally rejected.
+MAX_CONCURRENT_UPLOADS=1
+MAX_IN_FLIGHT_UPLOAD_MB=151
 MAX_IMPORT_JOBS_PER_RUN=5
 MAX_IMPORT_BYTES_PER_RUN=157286400
 CHUNK_SIZE=2000

.gitleaksignore

@@ -16,3 +16,6 @@ b65acddaa13e69e0ad2a49783116056b3a1a3639:data/medications-snapshot.json:generic-
 # and an earlier allowlist comment that quoted the pattern are both pinned below.
 d7cfc5559f0d7a5db74eb49808f237c989b4136a:scripts/compare-retrieval-eval.ts:generic-api-key:37
 d2b7b89852a2090d56ce6aed1dd39417aedf332d:.gitleaksignore:generic-api-key:13
+# False positive: two synthetic chunk identifiers in a citation-array assertion
+# were parsed as a generic API key in the original remediation commit.
+b30bcb030edce84e2a0704a8714ee3ef1dcf642d:tests/rag-offline-answer.test.ts:generic-api-key:205

docs/site-map.md

@@ -16,7 +16,7 @@ This file is generated by `npm run sitemap:update`. Run `npm run sitemap:check`
 - `/favourites` - Saved clinical items and sets. Source: `src/app/favourites/page.tsx`.
 - `/forms` - Forms home and search surface. Source: `src/app/forms/page.tsx`.
 - `/medications` - Medication index redirect. Source: `src/app/medications/page.tsx`.
-- `/privacy` - Route discovered from app directory Source: `src/app/privacy/page.tsx`.
+- `/privacy` - Privacy and data-processing governance draft. Source: `src/app/privacy/page.tsx`.
 - `/reference/colour-coding` - Route discovered from app directory Source: `src/app/reference/colour-coding/page.tsx`.
 - `/services` - Services home and search surface. Source: `src/app/services/page.tsx`.
 

scripts/compare-retrieval-eval.ts

@@ -44,6 +44,21 @@ const METRIC_SPECS: MetricSpec[] = [
   },
   { name: "latency_failed_cases", field: "latency_failed_cases", kind: "array", required: false, digits: 0 },
   { name: "index_units_layer_count", field: "index_units", kind: "layer", required: false, digits: 0 },
+  { name: "ndcg_at_10", field: "ndcg_at_10", kind: "number", required: false, digits: 4 },
+  {
+    name: "irrelevant_source_rate_at_10",
+    field: "irrelevant_source_rate_at_10",
+    kind: "number",
+    required: false,
+    digits: 4,
+  },
+  {
+    name: "required_signal_coverage_at_10",
+    field: "required_signal_coverage_at_10",
+    kind: "number",
+    required: false,
+    digits: 4,
+  },
 ];
 
 type MetricValue = { present: boolean; value: number };

scripts/eval-quality.ts

@@ -677,6 +677,10 @@ ${markdownTable([
   ["Document recall@5", retrieval.document_recall_at_5],
   ["Content recall@5", retrieval.content_recall_at_5],
   ["MRR@10", retrieval.mrr_at_10],
+  ["nDCG@10", retrieval.ndcg_at_10],
+  ["Irrelevant source rate@10", retrieval.irrelevant_source_rate_at_10],
+  ["Required signal coverage@10", retrieval.required_signal_coverage_at_10],
+  ["Signal metric cases", retrieval.signal_metric_case_count],
   ["Content MRR@10", retrieval.content_mrr_at_10],
   ["Content MRR cases", retrieval.content_mrr_case_count],
   ["Median latency ms", retrieval.median_latency_ms],

scripts/eval-rag-offline.mjs

@@ -2,6 +2,7 @@
 import { spawnSync } from "node:child_process";
 import { readFileSync } from "node:fs";
 import { resolve } from "node:path";
+import { validateOfflineContractTests } from "./rag-offline-contract.mjs";
 
 const goldenPath = "scripts/fixtures/rag-retrieval-golden.json";
 const cases = JSON.parse(readFileSync(goldenPath, "utf8"));
@@ -68,13 +69,14 @@ if (failures.length > 0) {
 
 console.log(`Offline RAG fixture schema passed (${cases.length} golden retrieval cases).`);
 
-const contractTests = [
-  "tests/rag-offline-answer.test.ts",
-  "tests/rag-answer-fallback.test.ts",
-  "tests/retrieval-selection.test.ts",
-  "tests/citations.test.ts",
-  "tests/smart-rag-api.test.ts",
-];
+const contractTestsPath = "scripts/fixtures/rag-offline-contract-tests.json";
+const contractTests = JSON.parse(readFileSync(contractTestsPath, "utf8"));
+const contractTestFailures = validateOfflineContractTests(contractTests);
+if (contractTestFailures.length > 0) {
+  console.error(`${contractTestsPath} failed the offline safety contract:`);
+  for (const failure of contractTestFailures) console.error(`- ${failure}`);
+  process.exit(1);
+}
 const vitestPath = resolve("node_modules/vitest/vitest.mjs");
 const offlineEnv = { ...process.env };
 for (const key of [

scripts/eval-retrieval.ts

@@ -67,6 +67,10 @@ export type GoldenRetrievalResult = {
   topK: number;
   reciprocalRankAt10: number;
   contentReciprocalRankAt10: number;
+  declaredSignalCount: number;
+  ndcgAt10: number;
+  irrelevantSourceRateAt10: number;
+  requiredSignalCoverageAt10: number;
   latencyMs: number;
   retrievalStrategy: string | null;
   retrievalPlan: string | null;
@@ -361,16 +365,24 @@ function expectedContentHits(
   results: SearchResult[],
   limit: number,
 ) {
-  const topContentText = results.slice(0, limit).map(resultContentText).join(" ");
+  const topResults = results.slice(0, limit);
   const hits = expectedTerms.filter((expectation) =>
-    contentExpectationAlternatives(expectation).some((term) => textContainsClinicalTerm(topContentText, term)),
+    topResults.some((result) => contentExpectationMatchesResult(expectation, result)),
   );
   return {
     hits: hits.map(contentExpectationLabel),
     missing: expectedTerms.filter((expectation) => !hits.includes(expectation)).map(contentExpectationLabel),
   };
 }
 
+function contentExpectationMatchesResult(
+  expectation: GoldenRetrievalCase["expectedContentTerms"][number],
+  result: SearchResult,
+) {
+  const contentText = resultContentText(result);
+  return contentExpectationAlternatives(expectation).some((term) => textContainsClinicalTerm(contentText, term));
+}
+
 function reciprocalRankAt10(expectedSubstrings: string[], results: SearchResult[]) {
   if (expectedSubstrings.length === 0) return 0;
   const index = results.slice(0, 10).findIndex((result) =>
@@ -410,6 +422,60 @@ function contentReciprocalRankAt10(
   return total / expectedTerms.length;
 }
 
+type RetrievalSignalMetrics = {
+  declaredSignalCount: number;
+  ndcgAt10: number;
+  irrelevantSourceRateAt10: number;
+  requiredSignalCoverageAt10: number;
+};
+
+/**
+ * Scores only signals explicitly declared by a golden case. Each document expectation,
+ * content expectation group, and required table-evidence flag is one signal. A result's
+ * graded relevance is the number of those signals it carries; nDCG uses the ideal ordering
+ * of the observed grades, so it measures ordering independently from coverage.
+ */
+function retrievalSignalMetricsAt10(testCase: GoldenRetrievalCase, results: SearchResult[]): RetrievalSignalMetrics {
+  const top = results.slice(0, 10);
+  const documentSignals = testCase.expectedDocumentSubstrings.map((expectation) => (result: SearchResult) => {
+    const text =
+      !clinicalDocumentAliases[expectation] && !/\.pdf$/i.test(expectation)
+        ? resultDocumentEvidenceText(result)
+        : resultDocumentText(result);
+    return documentExpectationAlternatives(expectation).some((alternative) => text.includes(alternative));
+  });
+  const contentSignals = testCase.expectedContentTerms.map(
+    (expectation) => (result: SearchResult) => contentExpectationMatchesResult(expectation, result),
+  );
+  const signals = [
+    ...documentSignals,
+    ...contentSignals,
+    ...(testCase.expectTableEvidence ? [(result: SearchResult) => hasTableEvidence([result], 1)] : []),
+  ];
+  const declaredSignalCount = signals.length;
+  if (declaredSignalCount === 0) {
+    return {
+      declaredSignalCount: 0,
+      ndcgAt10: 1,
+      irrelevantSourceRateAt10: 0,
+      requiredSignalCoverageAt10: 1,
+    };
+  }
+
+  const grades = top.map((result) => signals.filter((matches) => matches(result)).length);
+  const dcg = (values: number[]) =>
+    values.reduce((sum, grade, index) => sum + (2 ** grade - 1) / Math.log2(index + 2), 0);
+  const idealDcg = dcg([...grades].sort((left, right) => right - left));
+  const matchedSignals = signals.filter((matches) => top.some((result) => matches(result))).length;
+
+  return {
+    declaredSignalCount,
+    ndcgAt10: idealDcg > 0 ? dcg(grades) / idealDcg : 0,
+    irrelevantSourceRateAt10: top.length > 0 ? grades.filter((grade) => grade === 0).length / top.length : 0,
+    requiredSignalCoverageAt10: matchedSignals / declaredSignalCount,
+  };
+}
+
 export function retrievalLimitForGoldenCase(testCase: GoldenRetrievalCase) {
   return Math.max(testCase.topK, 10);
 }
@@ -528,6 +594,7 @@ export function evaluateGoldenRetrievalCase(args: {
       : contentHits.hits.length / args.testCase.expectedContentTerms.length;
   const tableEvidenceFound = hasTableEvidence(args.results, 5);
   const tableEvidenceFoundAtK = hasTableEvidence(args.results, topK);
+  const signalMetrics = retrievalSignalMetricsAt10(args.testCase, args.results);
   const actualQueryClass = args.telemetry.query_class ?? null;
   const failures: string[] = [];
   const vectorLayerCount = Object.entries(args.telemetry.retrieval_layer_counts ?? {}).reduce(
@@ -581,6 +648,7 @@ export function evaluateGoldenRetrievalCase(args: {
     topK,
     reciprocalRankAt10: reciprocalRankAt10(args.testCase.expectedDocumentSubstrings, args.results),
     contentReciprocalRankAt10: contentReciprocalRankAt10(args.testCase.expectedContentTerms, args.results),
+    ...signalMetrics,
     latencyMs: args.latencyMs,
     retrievalStrategy: args.telemetry.retrieval_strategy ?? null,
     retrievalPlan: args.telemetry.retrieval_plan ?? null,
@@ -645,6 +713,15 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
     return counts;
   }, {});
   const forceEmbeddingResults = results.filter((result) => result.forceEmbedding);
+  const signalMetricResults = results.filter((result) => result.declaredSignalCount > 0);
+  const signalMetricAverage = (select: (result: GoldenRetrievalResult) => number, neutral: number) =>
+    signalMetricResults.length > 0
+      ? Number(
+          (signalMetricResults.reduce((sum, result) => sum + select(result), 0) / signalMetricResults.length).toFixed(
+            4,
+          ),
+        )
+      : neutral;
   return {
     case_count: results.length,
     document_recall_at_5: Number(
@@ -664,6 +741,10 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
       ).toFixed(4),
     ),
     content_mrr_case_count: contentRankCases.length,
+    ndcg_at_10: signalMetricAverage((result) => result.ndcgAt10, 1),
+    irrelevant_source_rate_at_10: signalMetricAverage((result) => result.irrelevantSourceRateAt10, 0),
+    required_signal_coverage_at_10: signalMetricAverage((result) => result.requiredSignalCoverageAt10, 1),
+    signal_metric_case_count: signalMetricResults.length,
     median_latency_ms: percentile(
       results.map((result) => result.latencyMs),
       50,
@@ -753,6 +834,9 @@ function printHumanSummary(summary: ReturnType<typeof summarizeGoldenRetrievalRe
   console.log(
     `  content_mrr@10=${summary.content_mrr_at_10} (over ${summary.content_mrr_case_count} content-term case(s))`,
   );
+  console.log(`  ndcg@10=${summary.ndcg_at_10} (over ${summary.signal_metric_case_count} signal case(s))`);
+  console.log(`  irrelevant_source_rate@10=${summary.irrelevant_source_rate_at_10}`);
+  console.log(`  required_signal_coverage@10=${summary.required_signal_coverage_at_10}`);
   console.log(`  median_latency_ms=${summary.median_latency_ms}`);
   console.log(`  p90_latency_ms=${summary.p90_latency_ms}`);
   console.log(`  retrieval_strategy_counts=${JSON.stringify(summary.retrieval_strategy_counts)}`);
@@ -903,7 +987,7 @@ async function main() {
             ? "FAIL"
             : "PASS";
       console.log(
-        `${status} ${result.id} hit@${result.topK}=${result.hitAtK ? "1" : "0"} docRecall@5=${result.documentRecallAt5.toFixed(2)} contentRecall@5=${result.contentRecallAt5.toFixed(2)} rr@10=${result.reciprocalRankAt10.toFixed(2)} contentRR@10=${result.contentReciprocalRankAt10.toFixed(2)} latency=${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} gate=${result.coverageGateReason ?? "none"} layers=${JSON.stringify(result.retrievalLayerCounts ?? {})}`,
+        `${status} ${result.id} hit@${result.topK}=${result.hitAtK ? "1" : "0"} docRecall@5=${result.documentRecallAt5.toFixed(2)} contentRecall@5=${result.contentRecallAt5.toFixed(2)} rr@10=${result.reciprocalRankAt10.toFixed(2)} contentRR@10=${result.contentReciprocalRankAt10.toFixed(2)} ndcg@10=${result.ndcgAt10.toFixed(2)} irrelevant@10=${result.irrelevantSourceRateAt10.toFixed(2)} signalCoverage@10=${result.requiredSignalCoverageAt10.toFixed(2)} latency=${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} gate=${result.coverageGateReason ?? "none"} layers=${JSON.stringify(result.retrievalLayerCounts ?? {})}`,
       );
     }
   }

scripts/fixtures/rag-offline-contract-tests.json

@@ -0,0 +1,23 @@
+[
+  "tests/rag-offline-contract.test.ts",
+  "tests/rag-offline-answer.test.ts",
+  "tests/rag-answer-fallback.test.ts",
+  "tests/retrieval-selection.test.ts",
+  "tests/citations.test.ts",
+  "tests/smart-rag-api.test.ts",
+  "tests/deep-memory.test.ts",
+  "tests/deep-memory-transaction-sql.test.ts",
+  "tests/retrieval-access-scope.test.ts",
+  "tests/retrieval-hydration-scope.test.ts",
+  "tests/answer-verification.test.ts",
+  "tests/rag-routing.test.ts",
+  "tests/cross-document-synthesis.test.ts",
+  "tests/rag-comparison.test.ts",
+  "tests/rag-claim-support.test.ts",
+  "tests/answer-render-policy.test.ts",
+  "tests/canonical-answer-table.test.ts",
+  "tests/privacy.test.ts",
+  "tests/private-rag-access.test.ts",
+  "tests/upload-admission.test.ts",
+  "tests/privacy-ui.test.ts"
+]

scripts/generate-site-map.ts

@@ -48,6 +48,7 @@ const routeDescriptions: Record<string, string> = {
   "/forms/[slug]": "Registry-backed form detail.",
   "/medications": "Medication index redirect.",
   "/medications/[slug]": "Medication detail.",
+  "/privacy": "Privacy and data-processing governance draft.",
   "/services": "Services home and search surface.",
   "/services/[slug]": "Registry-backed service detail.",
 };

scripts/rag-offline-contract.mjs

@@ -0,0 +1,38 @@
+export const requiredOfflineContractTests = Object.freeze([
+  "tests/rag-offline-contract.test.ts",
+  "tests/rag-offline-answer.test.ts",
+  "tests/rag-answer-fallback.test.ts",
+  "tests/retrieval-selection.test.ts",
+  "tests/citations.test.ts",
+  "tests/smart-rag-api.test.ts",
+  "tests/deep-memory.test.ts",
+  "tests/deep-memory-transaction-sql.test.ts",
+  "tests/retrieval-access-scope.test.ts",
+  "tests/retrieval-hydration-scope.test.ts",
+  "tests/answer-verification.test.ts",
+  "tests/rag-routing.test.ts",
+  "tests/cross-document-synthesis.test.ts",
+  "tests/rag-comparison.test.ts",
+  "tests/rag-claim-support.test.ts",
+  "tests/answer-render-policy.test.ts",
+  "tests/canonical-answer-table.test.ts",
+  "tests/privacy.test.ts",
+  "tests/private-rag-access.test.ts",
+  "tests/upload-admission.test.ts",
+  "tests/privacy-ui.test.ts",
+]);
+
+export function validateOfflineContractTests(suites) {
+  const failures = [];
+  if (!Array.isArray(suites) || suites.length === 0) {
+    return ["contract suite list must be a non-empty array"];
+  }
+  if (!suites.every((suite) => typeof suite === "string" && suite.trim().length > 0)) {
+    failures.push("contract suite list contains an invalid path");
+  }
+  if (new Set(suites).size !== suites.length) failures.push("contract suite list contains duplicates");
+  for (const required of requiredOfflineContractTests) {
+    if (!suites.includes(required)) failures.push(`missing required suite: ${required}`);
+  }
+  return failures;
+}

src/app/api/answer/route.ts

@@ -1,7 +1,7 @@
 import { NextResponse } from "next/server";
 import { z } from "zod";
 import { demoAnswer } from "@/lib/demo-data";
-import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
+import { isDemoMode } from "@/lib/env";
 import { answerQuestionWithScope } from "@/lib/rag";
 import { jsonError, PublicApiError } from "@/lib/http";
 import {
@@ -14,6 +14,7 @@ import { classifyRagQuery } from "@/lib/clinical-search";
 import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
 import { clinicalQueryModeSchema, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode";
 import { resolveSearchScope, searchScopeFiltersSchema } from "@/lib/search-scope";
+import { resolveRetrievalAccessScope } from "@/lib/owner-scope";
 import {
   hasDangerSourceGovernanceWarning,
   sourceGovernanceRefusalAnswer,
@@ -36,7 +37,6 @@ const answerSchema = z.object({
   documentIds: z.array(z.string().uuid()).max(25).optional(),
   filters: searchScopeFiltersSchema.optional(),
   queryMode: clinicalQueryModeSchema.optional().default("auto"),
-  skipCache: z.boolean().optional().default(false),
 });
 
 type AnswerRequestBody = z.infer<typeof answerSchema>;
@@ -82,7 +82,7 @@ export async function POST(request: Request) {
 
     const supabase = createAdminClient();
     const access = await publicAccessContext(request, supabase);
-    const publicOnly = !access.authenticated && !isLocalNoAuthMode();
+    const accessScope = resolveRetrievalAccessScope(access.ownerId);
 
     const rateLimit = await consumeSubjectApiRateLimit({
       supabase,
@@ -96,8 +96,7 @@ export async function POST(request: Request) {
 
     const scope = await resolveSearchScope({
       supabase,
-      ownerId: access.ownerId,
-      publicOnly,
+      accessScope,
       documentIds: answerBody.documentIds ?? (answerBody.documentId ? [answerBody.documentId] : undefined),
       filters: answerBody.filters,
     });
@@ -127,9 +126,9 @@ export async function POST(request: Request) {
           answerBody.documentIds ??
           (answerBody.documentId ? [answerBody.documentId] : undefined)),
       ownerId: access.ownerId,
+      accessScope,
       allowGlobalSearch: !access.ownerId,
       queryMode: answerBody.queryMode,
-      skipCache: answerBody.skipCache,
       signal: request.signal,
     });
     const warnings = sourceGovernanceWarnings({