feat(worker): flag-gated medspaCy assertion tagging + offline eval gate

.env.example

@@ -132,6 +132,12 @@ WORKER_MAX_CAPTIONED_IMAGES_PER_DOCUMENT=15
 WORKER_MAX_CAPTIONED_IMAGES_PER_PAGE=2
 WORKER_VISION_CONCURRENCY=4
 WORKER_INLINE_ENRICHMENT=false
+# Eval-gated experiment: tag chunk metadata with medspaCy assertion status
+# (negated/uncertain/family/historical) at ingestion. Requires the optional
+# medspacy Python dependency (worker/python/requirements.txt) and a reviewed
+# `npm run eval:assertions` run before enabling. Fail-open; nothing consumes
+# the annotations yet.
+WORKER_MEDSPACY_ASSERTION=false
 PYTHON_BIN=python
 # Optional when Tesseract is installed outside PATH on Windows.
 TESSERACT_CMD=C:\Program Files\Tesseract-OCR\tesseract.exe

package.json

@@ -110,6 +110,7 @@
     "eval:retrieval:index-unit-vector": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality --force-embedding --json-out artifacts/retrieval/index-unit-vector.json",
     "eval:retrieval:compare": "node scripts/run-tsx.mjs scripts/compare-retrieval-eval.ts",
     "eval:retrieval:latency": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode latency --case-timeout-ms 25000 --p90-ms 20000",
+    "eval:assertions": "node scripts/run-tsx.mjs scripts/eval-assertions.ts",
     "eval:search": "node scripts/run-tsx.mjs scripts/eval-search.ts",
     "eval:search:api": "node scripts/run-tsx.mjs scripts/eval-search-api.ts",
     "retrieval:health": "node scripts/run-tsx.mjs scripts/retrieval-health.ts",

scripts/eval-assertions.ts

@@ -0,0 +1,175 @@
+// Offline eval for medspaCy assertion tagging (worker/python/analyze_assertions.py).
+// No providers, no Supabase — local Python only. This is the acceptance gate that must
+// be reviewed before WORKER_MEDSPACY_ASSERTION is ever enabled, and before any future
+// PR consumes chunk metadata.assertion in the answer pipeline.
+//
+// Usage: npm run eval:assertions [-- --fixture path --min-accuracy 0.8 --json-out path]
+
+import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import { annotateChunkAssertions } from "../worker/assertion-tagging";
+
+const expectationSchema = z.object({
+  negated: z.array(z.string()).default([]),
+  uncertain: z.array(z.string()).default([]),
+  family: z.array(z.string()).default([]),
+  historical: z.array(z.string()).default([]),
+  asserted: z.array(z.string()).default([]),
+});
+
+const goldenCaseSchema = z.object({
+  id: z.string(),
+  text: z.string().min(1),
+  targets: z.array(z.string()).min(1),
+  // Free-text rationale for non-obvious expectations (JSON has no comments).
+  note: z.string().optional(),
+  expect: expectationSchema,
+});
+
+const fixtureSchema = z.object({
+  description: z.string().optional(),
+  cases: z.array(goldenCaseSchema).min(1),
+});
+
+type Check = {
+  caseId: string;
+  category: "negated" | "uncertain" | "family" | "historical" | "no-false-negation" | "no-false-uncertainty";
+  term: string;
+  pass: boolean;
+  detail: string;
+};
+
+function argValue(flag: string): string | undefined {
+  const index = process.argv.indexOf(flag);
+  return index === -1 ? undefined : process.argv[index + 1];
+}
+
+function normalizeTerm(term: string) {
+  return term.trim().toLowerCase();
+}
+
+async function main() {
+  const fixturePath = argValue("--fixture") ?? path.join("scripts", "fixtures", "assertion-golden.json");
+  const minAccuracy = Number(argValue("--min-accuracy") ?? "0.8");
+  const jsonOut = argValue("--json-out");
+  // A mistyped threshold must fail loudly, never silently disable the gate:
+  // accuracy < NaN is always false, which would report PASS at any score.
+  if (!Number.isFinite(minAccuracy) || minAccuracy < 0 || minAccuracy > 1) {
+    throw new Error(`--min-accuracy must be a number between 0 and 1 (got "${argValue("--min-accuracy")}").`);
+  }
+
+  const fixture = fixtureSchema.parse(JSON.parse(await readFile(fixturePath, "utf8")));
+
+  // One Python invocation for the whole fixture: union the targets (extra targets are
+  // harmless — they only match if present in a text) and key chunks by case id.
+  const targets = Array.from(new Set(fixture.cases.flatMap((c) => c.targets.map(normalizeTerm))));
+  const assertions = await annotateChunkAssertions(
+    fixture.cases.map((c) => ({ id: c.id, text: c.text })),
+    targets,
+  );
+  if (assertions.size === 0) {
+    console.error(
+      "Assertion tagging returned no results — is medspacy installed? (pip install -r worker/python/requirements.txt)",
+    );
+    process.exit(1);
+  }
+
+  const checks: Check[] = [];
+  for (const goldenCase of fixture.cases) {
+    const tagged = assertions.get(goldenCase.id);
+    const negated = (tagged?.negated_terms ?? []).map(normalizeTerm);
+    const uncertain = (tagged?.uncertain_terms ?? []).map(normalizeTerm);
+    const family = (tagged?.family_terms ?? []).map(normalizeTerm);
+    const historical = (tagged?.historical_terms ?? []).map(normalizeTerm);
+
+    const membership: Array<[Check["category"], string[], string[]]> = [
+      ["negated", goldenCase.expect.negated, negated],
+      ["uncertain", goldenCase.expect.uncertain, uncertain],
+      ["family", goldenCase.expect.family, family],
+      ["historical", goldenCase.expect.historical, historical],
+    ];
+    for (const [category, expected, actual] of membership) {
+      for (const term of expected.map(normalizeTerm)) {
+        const pass = actual.includes(term);
+        checks.push({
+          caseId: goldenCase.id,
+          category,
+          term,
+          pass,
+          detail: pass ? "ok" : `expected "${term}" in ${category}_terms, got [${actual.join(", ")}]`,
+        });
+      }
+    }
+    // False-positive gate: EVERY target the case does not expect in an
+    // existence-changing category (negated/uncertain) must be absent from it —
+    // an over-broad ConText rule must fail here, not pass silently. Family and
+    // historical stay positive-only: they are contextual qualifiers, and cases
+    // may legitimately overlap them with asserted mentions.
+    const expectedNegated = new Set(goldenCase.expect.negated.map(normalizeTerm));
+    const expectedUncertain = new Set(goldenCase.expect.uncertain.map(normalizeTerm));
+    for (const term of goldenCase.targets.map(normalizeTerm)) {
+      if (!expectedNegated.has(term)) {
+        const pass = !negated.includes(term);
+        checks.push({
+          caseId: goldenCase.id,
+          category: "no-false-negation",
+          term,
+          pass,
+          detail: pass ? "ok" : `target "${term}" wrongly flagged negated`,
+        });
+      }
+      if (!expectedUncertain.has(term)) {
+        const pass = !uncertain.includes(term);
+        checks.push({
+          caseId: goldenCase.id,
+          category: "no-false-uncertainty",
+          term,
+          pass,
+          detail: pass ? "ok" : `target "${term}" wrongly flagged uncertain`,
+        });
+      }
+    }
+  }
+
+  const byCategory = new Map<string, { pass: number; total: number }>();
+  for (const check of checks) {
+    const bucket = byCategory.get(check.category) ?? { pass: 0, total: 0 };
+    bucket.total += 1;
+    if (check.pass) bucket.pass += 1;
+    byCategory.set(check.category, bucket);
+  }
+  const failures = checks.filter((check) => !check.pass);
+  const accuracy = checks.length === 0 ? 0 : (checks.length - failures.length) / checks.length;
+
+  console.log("\nAssertion tagging eval");
+  console.log("======================");
+  for (const [category, bucket] of byCategory) {
+    console.log(`${category.padEnd(11)} ${bucket.pass}/${bucket.total}`);
+  }
+  console.log(`overall     ${(accuracy * 100).toFixed(1)}% (min ${(minAccuracy * 100).toFixed(0)}%)`);
+  if (failures.length > 0) {
+    console.log("\nFailures:");
+    for (const failure of failures) {
+      console.log(`- [${failure.caseId}] ${failure.detail}`);
+    }
+  }
+
+  if (jsonOut) {
+    await writeFile(jsonOut, JSON.stringify({ accuracy, minAccuracy, checks }, null, 2), "utf8");
+    console.log(`\nWrote ${jsonOut}`);
+  }
+
+  if (accuracy < minAccuracy) {
+    console.error(
+      `\nFAIL: accuracy ${(accuracy * 100).toFixed(1)}% below threshold ${(minAccuracy * 100).toFixed(0)}%.`,
+    );
+    process.exit(1);
+  }
+  console.log("\nPASS");
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});

scripts/fixtures/assertion-golden.json

@@ -0,0 +1,186 @@
+{
+  "description": "Golden cases for medspaCy assertion tagging (worker/python/analyze_assertions.py). AU community mental health phrasing. Contract: expect.negated/uncertain/family/historical assert membership in the corresponding *_terms output; additionally EVERY target not listed in expect.negated (resp. expect.uncertain) must be absent from negated_terms (resp. uncertain_terms), so false-positive existence flags fail the gate. `expect.asserted` is documentation of intent — those terms are enforced by the universal false-positive rule. Family/historical are positive-only (contextual qualifiers may legitimately co-occur with asserted mentions).",
+  "cases": [
+    {
+      "id": "denies-si",
+      "text": "On review the patient denies suicidal ideation and denies any plan or intent.",
+      "targets": ["suicidal ideation"],
+      "expect": { "negated": ["suicidal ideation"] }
+    },
+    {
+      "id": "no-self-harm",
+      "text": "No thoughts of self harm were reported during the assessment.",
+      "targets": ["self harm"],
+      "expect": { "negated": ["self harm"] }
+    },
+    {
+      "id": "nil-hallucinations",
+      "text": "Nil auditory hallucinations at present.",
+      "targets": ["auditory hallucinations"],
+      "expect": { "negated": ["auditory hallucinations"] }
+    },
+    {
+      "id": "denies-avh",
+      "text": "She denies auditory hallucinations but describes ongoing paranoid ideation.",
+      "targets": ["auditory hallucinations", "paranoid ideation"],
+      "expect": { "negated": ["auditory hallucinations"], "asserted": ["paranoid ideation"] }
+    },
+    {
+      "id": "no-chest-pain-but",
+      "text": "No chest pain, however the patient reports palpitations since starting clozapine.",
+      "targets": ["chest pain", "palpitations", "clozapine"],
+      "expect": { "negated": ["chest pain"], "asserted": ["palpitations", "clozapine"] }
+    },
+    {
+      "id": "without-eps",
+      "text": "The dose was increased without extrapyramidal side effects.",
+      "targets": ["extrapyramidal side effects"],
+      "expect": { "negated": ["extrapyramidal side effects"] }
+    },
+    {
+      "id": "no-nms-signs",
+      "text": "There were no signs of neuroleptic malignant syndrome on examination.",
+      "targets": ["neuroleptic malignant syndrome"],
+      "expect": { "negated": ["neuroleptic malignant syndrome"] }
+    },
+    {
+      "id": "denies-substance",
+      "text": "He denies current alcohol use and denies illicit substance use.",
+      "targets": ["alcohol use", "substance use"],
+      "expect": { "negated": ["alcohol use", "substance use"] }
+    },
+    {
+      "id": "no-neutropenia",
+      "text": "Weekly FBC shows no neutropenia and the absolute neutrophil count remains within range.",
+      "targets": ["neutropenia", "absolute neutrophil count"],
+      "expect": { "negated": ["neutropenia"], "asserted": ["absolute neutrophil count"] }
+    },
+    {
+      "id": "not-suicidal",
+      "text": "The consumer is not currently suicidal and was discharged with a safety plan.",
+      "targets": ["suicidal", "safety plan"],
+      "expect": { "negated": ["suicidal"], "asserted": ["safety plan"] }
+    },
+    {
+      "id": "possible-nms",
+      "text": "Possible neuroleptic malignant syndrome; cease antipsychotic and transfer to the emergency department.",
+      "targets": ["neuroleptic malignant syndrome", "antipsychotic"],
+      "expect": { "uncertain": ["neuroleptic malignant syndrome"] }
+    },
+    {
+      "id": "query-serotonin",
+      "text": "Impression: query serotonin syndrome given recent SSRI dose increase.",
+      "targets": ["serotonin syndrome", "ssri"],
+      "expect": { "uncertain": ["serotonin syndrome"] }
+    },
+    {
+      "id": "suspected-myocarditis",
+      "text": "Suspected myocarditis in week three of clozapine titration.",
+      "targets": ["myocarditis", "clozapine"],
+      "expect": { "uncertain": ["myocarditis"], "asserted": ["clozapine"] }
+    },
+    {
+      "id": "differential-psychosis",
+      "text": "Differential diagnosis includes drug induced psychosis versus first episode schizophrenia.",
+      "targets": ["psychosis", "schizophrenia"],
+      "expect": { "uncertain": ["psychosis", "schizophrenia"] }
+    },
+    {
+      "id": "may-have-akathisia",
+      "text": "The restlessness may represent akathisia rather than agitation.",
+      "targets": ["akathisia", "agitation"],
+      "note": "'rather than X' excludes X: agitation is correctly both negated (excluded) and uncertain (inside the tentative differential). Surfaced by the false-positive gate; encoded as intended behavior.",
+      "expect": { "uncertain": ["akathisia", "agitation"], "negated": ["agitation"] }
+    },
+    {
+      "id": "if-toxicity",
+      "text": "If lithium toxicity develops, withhold the next dose and obtain an urgent level.",
+      "targets": ["lithium toxicity"],
+      "expect": { "uncertain": ["lithium toxicity"] }
+    },
+    {
+      "id": "family-bipolar",
+      "text": "Family history of bipolar disorder in her mother.",
+      "targets": ["bipolar disorder"],
+      "expect": { "family": ["bipolar disorder"] }
+    },
+    {
+      "id": "father-schizophrenia",
+      "text": "Her father was diagnosed with schizophrenia in his twenties.",
+      "targets": ["schizophrenia"],
+      "expect": { "family": ["schizophrenia"] }
+    },
+    {
+      "id": "family-suicide",
+      "text": "There is a family history of suicide on the paternal side.",
+      "targets": ["suicide"],
+      "expect": { "family": ["suicide"] }
+    },
+    {
+      "id": "history-dsh",
+      "text": "Past history of deliberate self harm as an adolescent.",
+      "targets": ["deliberate self harm"],
+      "expect": { "historical": ["deliberate self harm"] }
+    },
+    {
+      "id": "previous-overdose",
+      "text": "History of overdose in 2019 requiring ICU admission.",
+      "targets": ["overdose"],
+      "expect": { "historical": ["overdose"] }
+    },
+    {
+      "id": "prior-catatonia",
+      "text": "Previous episode of catatonia responded well to electroconvulsive therapy.",
+      "targets": ["catatonia", "electroconvulsive therapy"],
+      "expect": { "historical": ["catatonia"] }
+    },
+    {
+      "id": "asserted-avh",
+      "text": "The patient reports auditory hallucinations commanding self harm.",
+      "targets": ["auditory hallucinations"],
+      "expect": { "asserted": ["auditory hallucinations"] }
+    },
+    {
+      "id": "asserted-clozapine",
+      "text": "Commenced on clozapine 12.5 mg with standard titration and weekly monitoring.",
+      "targets": ["clozapine", "monitoring"],
+      "expect": { "asserted": ["clozapine", "monitoring"] }
+    },
+    {
+      "id": "asserted-qtc",
+      "text": "The ECG shows a prolonged QTc of 480 ms; repeat after dose reduction.",
+      "targets": ["qtc", "ecg"],
+      "expect": { "asserted": ["qtc", "ecg"] }
+    },
+    {
+      "id": "asserted-si-active",
+      "text": "He expresses active suicidal ideation with a specific plan.",
+      "targets": ["suicidal ideation"],
+      "expect": { "asserted": ["suicidal ideation"] }
+    },
+    {
+      "id": "asserted-seclusion",
+      "text": "Seclusion was initiated at 2140 following a code black.",
+      "targets": ["seclusion", "code black"],
+      "expect": { "asserted": ["seclusion", "code black"] }
+    },
+    {
+      "id": "asserted-cto",
+      "text": "She remains on a community treatment order reviewed by the tribunal in March.",
+      "targets": ["community treatment order"],
+      "expect": { "asserted": ["community treatment order"] }
+    },
+    {
+      "id": "mixed-negation-scope",
+      "text": "No evidence of serotonin syndrome, although tremor and hyperreflexia should prompt urgent review.",
+      "targets": ["serotonin syndrome", "tremor"],
+      "expect": { "negated": ["serotonin syndrome"] }
+    },
+    {
+      "id": "mixed-family-vs-patient",
+      "text": "Family history of depression; the patient herself reports low mood and anhedonia.",
+      "targets": ["depression", "low mood"],
+      "expect": { "family": ["depression"], "asserted": ["low mood"] }
+    }
+  ]
+}

src/lib/env.ts

@@ -184,6 +184,15 @@ const envSchema = z.object({
     .enum(["true", "false"])
     .default("false")
     .transform((value) => value === "true"),
+  // Eval-gated experiment: tag chunk metadata with medspaCy ConText assertion status
+  // (negated/uncertain/family/historical) at ingestion. Default off — requires the
+  // optional medspacy Python dependency and a reviewed `npm run eval:assertions` run
+  // before enabling. Nothing consumes the annotations yet (answer-verification is
+  // deliberately NOT wired to them). See worker/assertion-tagging.ts.
+  WORKER_MEDSPACY_ASSERTION: z
+    .enum(["true", "false"])
+    .default("false")
+    .transform((value) => value === "true"),
   PYTHON_BIN: z.string().default("python"),
   NEXT_PUBLIC_DEMO_MODE: z.enum(["true", "false"]).optional().default("false"),
 });