diff --git a/.env.example b/.env.example index 9bb20861b..78096478f 100644 --- a/.env.example +++ b/.env.example @@ -132,6 +132,12 @@ WORKER_MAX_CAPTIONED_IMAGES_PER_DOCUMENT=15 WORKER_MAX_CAPTIONED_IMAGES_PER_PAGE=2 WORKER_VISION_CONCURRENCY=4 WORKER_INLINE_ENRICHMENT=false +# Eval-gated experiment: tag chunk metadata with medspaCy assertion status +# (negated/uncertain/family/historical) at ingestion. Requires the optional +# medspacy Python dependency (worker/python/requirements.txt) and a reviewed +# `npm run eval:assertions` run before enabling. Fail-open; nothing consumes +# the annotations yet. +WORKER_MEDSPACY_ASSERTION=false PYTHON_BIN=python # Optional when Tesseract is installed outside PATH on Windows. TESSERACT_CMD=C:\Program Files\Tesseract-OCR\tesseract.exe diff --git a/package.json b/package.json index 683023ce3..4abf8d5c7 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "eval:retrieval:index-unit-vector": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality --force-embedding --json-out artifacts/retrieval/index-unit-vector.json", "eval:retrieval:compare": "node scripts/run-tsx.mjs scripts/compare-retrieval-eval.ts", "eval:retrieval:latency": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode latency --case-timeout-ms 25000 --p90-ms 20000", + "eval:assertions": "node scripts/run-tsx.mjs scripts/eval-assertions.ts", "eval:search": "node scripts/run-tsx.mjs scripts/eval-search.ts", "eval:search:api": "node scripts/run-tsx.mjs scripts/eval-search-api.ts", "retrieval:health": "node scripts/run-tsx.mjs scripts/retrieval-health.ts", diff --git a/scripts/eval-assertions.ts b/scripts/eval-assertions.ts new file mode 100644 index 000000000..d83293ee9 --- /dev/null +++ b/scripts/eval-assertions.ts @@ -0,0 +1,175 @@ +// Offline eval for medspaCy assertion tagging (worker/python/analyze_assertions.py). +// No providers, no Supabase — local Python only. This is the acceptance gate that must +// be reviewed before WORKER_MEDSPACY_ASSERTION is ever enabled, and before any future +// PR consumes chunk metadata.assertion in the answer pipeline. +// +// Usage: npm run eval:assertions [-- --fixture path --min-accuracy 0.8 --json-out path] + +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { annotateChunkAssertions } from "../worker/assertion-tagging"; + +const expectationSchema = z.object({ + negated: z.array(z.string()).default([]), + uncertain: z.array(z.string()).default([]), + family: z.array(z.string()).default([]), + historical: z.array(z.string()).default([]), + asserted: z.array(z.string()).default([]), +}); + +const goldenCaseSchema = z.object({ + id: z.string(), + text: z.string().min(1), + targets: z.array(z.string()).min(1), + // Free-text rationale for non-obvious expectations (JSON has no comments). + note: z.string().optional(), + expect: expectationSchema, +}); + +const fixtureSchema = z.object({ + description: z.string().optional(), + cases: z.array(goldenCaseSchema).min(1), +}); + +type Check = { + caseId: string; + category: "negated" | "uncertain" | "family" | "historical" | "no-false-negation" | "no-false-uncertainty"; + term: string; + pass: boolean; + detail: string; +}; + +function argValue(flag: string): string | undefined { + const index = process.argv.indexOf(flag); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function normalizeTerm(term: string) { + return term.trim().toLowerCase(); +} + +async function main() { + const fixturePath = argValue("--fixture") ?? path.join("scripts", "fixtures", "assertion-golden.json"); + const minAccuracy = Number(argValue("--min-accuracy") ?? "0.8"); + const jsonOut = argValue("--json-out"); + // A mistyped threshold must fail loudly, never silently disable the gate: + // accuracy < NaN is always false, which would report PASS at any score. + if (!Number.isFinite(minAccuracy) || minAccuracy < 0 || minAccuracy > 1) { + throw new Error(`--min-accuracy must be a number between 0 and 1 (got "${argValue("--min-accuracy")}").`); + } + + const fixture = fixtureSchema.parse(JSON.parse(await readFile(fixturePath, "utf8"))); + + // One Python invocation for the whole fixture: union the targets (extra targets are + // harmless — they only match if present in a text) and key chunks by case id. + const targets = Array.from(new Set(fixture.cases.flatMap((c) => c.targets.map(normalizeTerm)))); + const assertions = await annotateChunkAssertions( + fixture.cases.map((c) => ({ id: c.id, text: c.text })), + targets, + ); + if (assertions.size === 0) { + console.error( + "Assertion tagging returned no results — is medspacy installed? (pip install -r worker/python/requirements.txt)", + ); + process.exit(1); + } + + const checks: Check[] = []; + for (const goldenCase of fixture.cases) { + const tagged = assertions.get(goldenCase.id); + const negated = (tagged?.negated_terms ?? []).map(normalizeTerm); + const uncertain = (tagged?.uncertain_terms ?? []).map(normalizeTerm); + const family = (tagged?.family_terms ?? []).map(normalizeTerm); + const historical = (tagged?.historical_terms ?? []).map(normalizeTerm); + + const membership: Array<[Check["category"], string[], string[]]> = [ + ["negated", goldenCase.expect.negated, negated], + ["uncertain", goldenCase.expect.uncertain, uncertain], + ["family", goldenCase.expect.family, family], + ["historical", goldenCase.expect.historical, historical], + ]; + for (const [category, expected, actual] of membership) { + for (const term of expected.map(normalizeTerm)) { + const pass = actual.includes(term); + checks.push({ + caseId: goldenCase.id, + category, + term, + pass, + detail: pass ? "ok" : `expected "${term}" in ${category}_terms, got [${actual.join(", ")}]`, + }); + } + } + // False-positive gate: EVERY target the case does not expect in an + // existence-changing category (negated/uncertain) must be absent from it — + // an over-broad ConText rule must fail here, not pass silently. Family and + // historical stay positive-only: they are contextual qualifiers, and cases + // may legitimately overlap them with asserted mentions. + const expectedNegated = new Set(goldenCase.expect.negated.map(normalizeTerm)); + const expectedUncertain = new Set(goldenCase.expect.uncertain.map(normalizeTerm)); + for (const term of goldenCase.targets.map(normalizeTerm)) { + if (!expectedNegated.has(term)) { + const pass = !negated.includes(term); + checks.push({ + caseId: goldenCase.id, + category: "no-false-negation", + term, + pass, + detail: pass ? "ok" : `target "${term}" wrongly flagged negated`, + }); + } + if (!expectedUncertain.has(term)) { + const pass = !uncertain.includes(term); + checks.push({ + caseId: goldenCase.id, + category: "no-false-uncertainty", + term, + pass, + detail: pass ? "ok" : `target "${term}" wrongly flagged uncertain`, + }); + } + } + } + + const byCategory = new Map(); + for (const check of checks) { + const bucket = byCategory.get(check.category) ?? { pass: 0, total: 0 }; + bucket.total += 1; + if (check.pass) bucket.pass += 1; + byCategory.set(check.category, bucket); + } + const failures = checks.filter((check) => !check.pass); + const accuracy = checks.length === 0 ? 0 : (checks.length - failures.length) / checks.length; + + console.log("\nAssertion tagging eval"); + console.log("======================"); + for (const [category, bucket] of byCategory) { + console.log(`${category.padEnd(11)} ${bucket.pass}/${bucket.total}`); + } + console.log(`overall ${(accuracy * 100).toFixed(1)}% (min ${(minAccuracy * 100).toFixed(0)}%)`); + if (failures.length > 0) { + console.log("\nFailures:"); + for (const failure of failures) { + console.log(`- [${failure.caseId}] ${failure.detail}`); + } + } + + if (jsonOut) { + await writeFile(jsonOut, JSON.stringify({ accuracy, minAccuracy, checks }, null, 2), "utf8"); + console.log(`\nWrote ${jsonOut}`); + } + + if (accuracy < minAccuracy) { + console.error( + `\nFAIL: accuracy ${(accuracy * 100).toFixed(1)}% below threshold ${(minAccuracy * 100).toFixed(0)}%.`, + ); + process.exit(1); + } + console.log("\nPASS"); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/fixtures/assertion-golden.json b/scripts/fixtures/assertion-golden.json new file mode 100644 index 000000000..ee047f980 --- /dev/null +++ b/scripts/fixtures/assertion-golden.json @@ -0,0 +1,186 @@ +{ + "description": "Golden cases for medspaCy assertion tagging (worker/python/analyze_assertions.py). AU community mental health phrasing. Contract: expect.negated/uncertain/family/historical assert membership in the corresponding *_terms output; additionally EVERY target not listed in expect.negated (resp. expect.uncertain) must be absent from negated_terms (resp. uncertain_terms), so false-positive existence flags fail the gate. `expect.asserted` is documentation of intent — those terms are enforced by the universal false-positive rule. Family/historical are positive-only (contextual qualifiers may legitimately co-occur with asserted mentions).", + "cases": [ + { + "id": "denies-si", + "text": "On review the patient denies suicidal ideation and denies any plan or intent.", + "targets": ["suicidal ideation"], + "expect": { "negated": ["suicidal ideation"] } + }, + { + "id": "no-self-harm", + "text": "No thoughts of self harm were reported during the assessment.", + "targets": ["self harm"], + "expect": { "negated": ["self harm"] } + }, + { + "id": "nil-hallucinations", + "text": "Nil auditory hallucinations at present.", + "targets": ["auditory hallucinations"], + "expect": { "negated": ["auditory hallucinations"] } + }, + { + "id": "denies-avh", + "text": "She denies auditory hallucinations but describes ongoing paranoid ideation.", + "targets": ["auditory hallucinations", "paranoid ideation"], + "expect": { "negated": ["auditory hallucinations"], "asserted": ["paranoid ideation"] } + }, + { + "id": "no-chest-pain-but", + "text": "No chest pain, however the patient reports palpitations since starting clozapine.", + "targets": ["chest pain", "palpitations", "clozapine"], + "expect": { "negated": ["chest pain"], "asserted": ["palpitations", "clozapine"] } + }, + { + "id": "without-eps", + "text": "The dose was increased without extrapyramidal side effects.", + "targets": ["extrapyramidal side effects"], + "expect": { "negated": ["extrapyramidal side effects"] } + }, + { + "id": "no-nms-signs", + "text": "There were no signs of neuroleptic malignant syndrome on examination.", + "targets": ["neuroleptic malignant syndrome"], + "expect": { "negated": ["neuroleptic malignant syndrome"] } + }, + { + "id": "denies-substance", + "text": "He denies current alcohol use and denies illicit substance use.", + "targets": ["alcohol use", "substance use"], + "expect": { "negated": ["alcohol use", "substance use"] } + }, + { + "id": "no-neutropenia", + "text": "Weekly FBC shows no neutropenia and the absolute neutrophil count remains within range.", + "targets": ["neutropenia", "absolute neutrophil count"], + "expect": { "negated": ["neutropenia"], "asserted": ["absolute neutrophil count"] } + }, + { + "id": "not-suicidal", + "text": "The consumer is not currently suicidal and was discharged with a safety plan.", + "targets": ["suicidal", "safety plan"], + "expect": { "negated": ["suicidal"], "asserted": ["safety plan"] } + }, + { + "id": "possible-nms", + "text": "Possible neuroleptic malignant syndrome; cease antipsychotic and transfer to the emergency department.", + "targets": ["neuroleptic malignant syndrome", "antipsychotic"], + "expect": { "uncertain": ["neuroleptic malignant syndrome"] } + }, + { + "id": "query-serotonin", + "text": "Impression: query serotonin syndrome given recent SSRI dose increase.", + "targets": ["serotonin syndrome", "ssri"], + "expect": { "uncertain": ["serotonin syndrome"] } + }, + { + "id": "suspected-myocarditis", + "text": "Suspected myocarditis in week three of clozapine titration.", + "targets": ["myocarditis", "clozapine"], + "expect": { "uncertain": ["myocarditis"], "asserted": ["clozapine"] } + }, + { + "id": "differential-psychosis", + "text": "Differential diagnosis includes drug induced psychosis versus first episode schizophrenia.", + "targets": ["psychosis", "schizophrenia"], + "expect": { "uncertain": ["psychosis", "schizophrenia"] } + }, + { + "id": "may-have-akathisia", + "text": "The restlessness may represent akathisia rather than agitation.", + "targets": ["akathisia", "agitation"], + "note": "'rather than X' excludes X: agitation is correctly both negated (excluded) and uncertain (inside the tentative differential). Surfaced by the false-positive gate; encoded as intended behavior.", + "expect": { "uncertain": ["akathisia", "agitation"], "negated": ["agitation"] } + }, + { + "id": "if-toxicity", + "text": "If lithium toxicity develops, withhold the next dose and obtain an urgent level.", + "targets": ["lithium toxicity"], + "expect": { "uncertain": ["lithium toxicity"] } + }, + { + "id": "family-bipolar", + "text": "Family history of bipolar disorder in her mother.", + "targets": ["bipolar disorder"], + "expect": { "family": ["bipolar disorder"] } + }, + { + "id": "father-schizophrenia", + "text": "Her father was diagnosed with schizophrenia in his twenties.", + "targets": ["schizophrenia"], + "expect": { "family": ["schizophrenia"] } + }, + { + "id": "family-suicide", + "text": "There is a family history of suicide on the paternal side.", + "targets": ["suicide"], + "expect": { "family": ["suicide"] } + }, + { + "id": "history-dsh", + "text": "Past history of deliberate self harm as an adolescent.", + "targets": ["deliberate self harm"], + "expect": { "historical": ["deliberate self harm"] } + }, + { + "id": "previous-overdose", + "text": "History of overdose in 2019 requiring ICU admission.", + "targets": ["overdose"], + "expect": { "historical": ["overdose"] } + }, + { + "id": "prior-catatonia", + "text": "Previous episode of catatonia responded well to electroconvulsive therapy.", + "targets": ["catatonia", "electroconvulsive therapy"], + "expect": { "historical": ["catatonia"] } + }, + { + "id": "asserted-avh", + "text": "The patient reports auditory hallucinations commanding self harm.", + "targets": ["auditory hallucinations"], + "expect": { "asserted": ["auditory hallucinations"] } + }, + { + "id": "asserted-clozapine", + "text": "Commenced on clozapine 12.5 mg with standard titration and weekly monitoring.", + "targets": ["clozapine", "monitoring"], + "expect": { "asserted": ["clozapine", "monitoring"] } + }, + { + "id": "asserted-qtc", + "text": "The ECG shows a prolonged QTc of 480 ms; repeat after dose reduction.", + "targets": ["qtc", "ecg"], + "expect": { "asserted": ["qtc", "ecg"] } + }, + { + "id": "asserted-si-active", + "text": "He expresses active suicidal ideation with a specific plan.", + "targets": ["suicidal ideation"], + "expect": { "asserted": ["suicidal ideation"] } + }, + { + "id": "asserted-seclusion", + "text": "Seclusion was initiated at 2140 following a code black.", + "targets": ["seclusion", "code black"], + "expect": { "asserted": ["seclusion", "code black"] } + }, + { + "id": "asserted-cto", + "text": "She remains on a community treatment order reviewed by the tribunal in March.", + "targets": ["community treatment order"], + "expect": { "asserted": ["community treatment order"] } + }, + { + "id": "mixed-negation-scope", + "text": "No evidence of serotonin syndrome, although tremor and hyperreflexia should prompt urgent review.", + "targets": ["serotonin syndrome", "tremor"], + "expect": { "negated": ["serotonin syndrome"] } + }, + { + "id": "mixed-family-vs-patient", + "text": "Family history of depression; the patient herself reports low mood and anhedonia.", + "targets": ["depression", "low mood"], + "expect": { "family": ["depression"], "asserted": ["low mood"] } + } + ] +} diff --git a/src/lib/env.ts b/src/lib/env.ts index 1ba265d8b..0f86b6d85 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -184,6 +184,15 @@ const envSchema = z.object({ .enum(["true", "false"]) .default("false") .transform((value) => value === "true"), + // Eval-gated experiment: tag chunk metadata with medspaCy ConText assertion status + // (negated/uncertain/family/historical) at ingestion. Default off — requires the + // optional medspacy Python dependency and a reviewed `npm run eval:assertions` run + // before enabling. Nothing consumes the annotations yet (answer-verification is + // deliberately NOT wired to them). See worker/assertion-tagging.ts. + WORKER_MEDSPACY_ASSERTION: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true"), PYTHON_BIN: z.string().default("python"), NEXT_PUBLIC_DEMO_MODE: z.enum(["true", "false"]).optional().default("false"), }); diff --git a/tests/assertion-tagging.test.ts b/tests/assertion-tagging.test.ts new file mode 100644 index 000000000..35116db64 --- /dev/null +++ b/tests/assertion-tagging.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi } from "vitest"; +import { + annotateChunkAssertions, + assertionMetadataValue, + defaultAssertionTargets, + parseAssertionPayload, + type AssertionScriptRunner, +} from "../worker/assertion-tagging"; + +// TS-side contract tests for flag-gated assertion tagging. The Python half +// (worker/python/analyze_assertions.py) is exercised by `npm run eval:assertions` +// locally — CI has no medspaCy, so every test here injects a fake runner. +// The load-bearing invariant is FAIL-OPEN: tagging must never throw into the +// ingestion job, whatever the subprocess does. + +const VALID_PAYLOAD = JSON.stringify({ + assertions: [ + { id: "0", negated_terms: ["chest pain"], uncertain_terms: [], family_terms: [], historical_terms: [] }, + { id: "1", negated_terms: [], uncertain_terms: ["serotonin syndrome"], family_terms: [], historical_terms: [] }, + ], + version: "1.3.1", + warnings: [], +}); + +const CHUNKS = [ + { id: "0", text: "No chest pain reported." }, + { id: "1", text: "Query serotonin syndrome." }, +]; +const TARGETS = ["chest pain", "serotonin syndrome"]; + +describe("parseAssertionPayload", () => { + it("parses a valid payload and defaults omitted arrays", () => { + const parsed = parseAssertionPayload(JSON.stringify({ assertions: [{ id: "0" }] })); + expect(parsed.assertions[0]).toEqual({ + id: "0", + negated_terms: [], + uncertain_terms: [], + family_terms: [], + historical_terms: [], + }); + expect(parsed.warnings).toEqual([]); + }); + + it("throws on non-JSON and on shape violations", () => { + expect(() => parseAssertionPayload("not json")).toThrow(); + expect(() => parseAssertionPayload(JSON.stringify({ assertions: [{ id: 5 }] }))).toThrow(); + }); +}); + +describe("annotateChunkAssertions", () => { + it("maps assertions by chunk id and stamps the medspaCy version", async () => { + const runner: AssertionScriptRunner = vi.fn(async () => VALID_PAYLOAD); + const result = await annotateChunkAssertions(CHUNKS, TARGETS, runner); + + expect(runner).toHaveBeenCalledWith({ chunks: CHUNKS, targets: TARGETS }); + expect(result.size).toBe(2); + expect(result.get("0")).toEqual({ + negated_terms: ["chest pain"], + uncertain_terms: [], + family_terms: [], + historical_terms: [], + medspacy_version: "1.3.1", + }); + expect(result.get("1")?.uncertain_terms).toEqual(["serotonin syndrome"]); + }); + + it("fails open to an empty map when the runner throws", async () => { + const runner: AssertionScriptRunner = vi.fn(async () => { + throw new Error("python exploded"); + }); + await expect(annotateChunkAssertions(CHUNKS, TARGETS, runner)).resolves.toEqual(new Map()); + }); + + it("fails open to an empty map on garbage output", async () => { + const runner: AssertionScriptRunner = vi.fn(async () => "Traceback (most recent call last): ..."); + await expect(annotateChunkAssertions(CHUNKS, TARGETS, runner)).resolves.toEqual(new Map()); + }); + + it("skips the subprocess entirely for empty chunks or targets", async () => { + const runner: AssertionScriptRunner = vi.fn(async () => VALID_PAYLOAD); + await expect(annotateChunkAssertions([], TARGETS, runner)).resolves.toEqual(new Map()); + await expect(annotateChunkAssertions(CHUNKS, [], runner)).resolves.toEqual(new Map()); + expect(runner).not.toHaveBeenCalled(); + }); +}); + +describe("assertionMetadataValue", () => { + it("normalizes a missing version to null for the jsonb column", () => { + const value = assertionMetadataValue( + { id: "0", negated_terms: ["x"], uncertain_terms: [], family_terms: [], historical_terms: [] }, + undefined, + ); + expect(value.medspacy_version).toBeNull(); + }); +}); + +describe("defaultAssertionTargets", () => { + it("supplies lowercase vocabulary terms and excludes typo entries", () => { + const targets = defaultAssertionTargets(); + expect(targets.length).toBeGreaterThan(50); + expect(targets).toContain("clozapine"); + expect(targets).toContain("neuroleptic malignant syndrome"); + expect(targets.every((term) => term === term.toLowerCase())).toBe(true); + }); +}); diff --git a/worker/assertion-tagging.ts b/worker/assertion-tagging.ts new file mode 100644 index 000000000..93d75c8ae --- /dev/null +++ b/worker/assertion-tagging.ts @@ -0,0 +1,157 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { z } from "zod"; +import { env } from "../src/lib/env"; +import { clinicalVocabularyEntries } from "../src/lib/clinical-vocabulary"; + +// Flag-gated (WORKER_MEDSPACY_ASSERTION) clinical assertion tagging. Chunk texts are +// batched through worker/python/analyze_assertions.py (medspaCy ConText) and the +// results land as a namespaced `assertion` key inside document_chunks.metadata jsonb. +// Nothing consumes the annotations yet — eval-first via `npm run eval:assertions`. +// Contract: FAIL-OPEN. Assertion tagging must never fail or block an ingestion job. + +export const chunkAssertionSchema = z.object({ + id: z.string(), + negated_terms: z.array(z.string()).default([]), + uncertain_terms: z.array(z.string()).default([]), + family_terms: z.array(z.string()).default([]), + historical_terms: z.array(z.string()).default([]), +}); + +export const assertionPayloadSchema = z.object({ + assertions: z.array(chunkAssertionSchema), + version: z.string().optional(), + warnings: z.array(z.string()).default([]), +}); + +export type ChunkAssertion = z.infer; +export type AssertionPayload = z.infer; +export type AssertionInputChunk = { id: string; text: string }; +export type AssertionMetadata = { + negated_terms: string[]; + uncertain_terms: string[]; + family_terms: string[]; + historical_terms: string[]; + medspacy_version: string | null; +}; + +/** Raw script output (file contents or stdout) -> validated payload. Throws on garbage. */ +export function parseAssertionPayload(raw: string): AssertionPayload { + return assertionPayloadSchema.parse(JSON.parse(raw)); +} + +export function assertionMetadataValue(assertion: ChunkAssertion, version?: string): AssertionMetadata { + return { + negated_terms: assertion.negated_terms, + uncertain_terms: assertion.uncertain_terms, + family_terms: assertion.family_terms, + historical_terms: assertion.historical_terms, + medspacy_version: version ?? null, + }; +} + +/** ConText only marks supplied targets; the worker tags the clinical vocabulary. */ +export function defaultAssertionTargets(): string[] { + const targets = new Set(); + for (const entry of clinicalVocabularyEntries()) { + if (entry.type === "typo") continue; + targets.add(entry.canonical.toLowerCase()); + for (const alias of entry.aliases) targets.add(alias.toLowerCase()); + } + return Array.from(targets); +} + +export type AssertionScriptRunner = (input: { chunks: AssertionInputChunk[]; targets: string[] }) => Promise; + +// Generous ceiling for one document's chunk batch (model load is seconds, tagging is +// fast). A hung Python process must reject — annotateChunkAssertions then fails open — +// so assertion tagging can never wedge an ingestion job. +const assertionScriptTimeoutMs = 120_000; + +function extractJsonFromStdout(stdout: string) { + const first = stdout.indexOf("{"); + const last = stdout.lastIndexOf("}"); + if (first === -1 || last === -1 || last < first) { + throw new Error("Assertion script produced no JSON output."); + } + return stdout.slice(first, last + 1); +} + +export const runAssertionScript: AssertionScriptRunner = async (input) => { + const scriptPath = path.join(process.cwd(), "worker", "python", "analyze_assertions.py"); + const workDir = await mkdtemp(path.join(tmpdir(), "clinical-kb-assertions-")); + const inputPath = path.join(workDir, "input.json"); + const outputPath = path.join(workDir, "output.json"); + try { + await writeFile(inputPath, JSON.stringify(input), "utf8"); + const stdout = await new Promise((resolve, reject) => { + const child = spawn(env.PYTHON_BIN, [scriptPath, inputPath, outputPath], { + stdio: ["ignore", "pipe", "pipe"], + }); + let out = ""; + let err = ""; + const timeout = setTimeout(() => { + child.kill(); + reject(new Error(`Assertion script timed out after ${assertionScriptTimeoutMs}ms.`)); + }, assertionScriptTimeoutMs); + child.stdout.on("data", (chunk) => { + out += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + err += chunk.toString(); + }); + child.on("error", (error) => { + clearTimeout(timeout); + reject(new Error(`Assertion script failed to start: ${error.message}`)); + }); + child.on("close", (code) => { + clearTimeout(timeout); + if (code !== 0) { + reject(new Error(err.trim() || `Assertion script exited with ${code}`)); + return; + } + resolve(out); + }); + }); + // File first (robust against stray prints), stdout JSON as fallback — + // same contract as the PDF extractor (src/lib/extractors/document.ts). + try { + return await readFile(outputPath, "utf8"); + } catch { + return extractJsonFromStdout(stdout); + } + } finally { + await rm(workDir, { recursive: true, force: true }); + } +}; + +/** + * Tag chunks with assertion metadata, keyed by chunk id. Fail-open: any script, + * parse, or validation failure logs a warning and returns an empty map so the + * ingestion job proceeds without annotations. + */ +export async function annotateChunkAssertions( + chunks: AssertionInputChunk[], + targets: string[], + runner: AssertionScriptRunner = runAssertionScript, +): Promise> { + if (chunks.length === 0 || targets.length === 0) return new Map(); + try { + const payload = parseAssertionPayload(await runner({ chunks, targets })); + for (const warning of payload.warnings) { + console.warn(`Assertion tagging warning: ${warning}`); + } + return new Map( + payload.assertions.map((assertion) => [assertion.id, assertionMetadataValue(assertion, payload.version)]), + ); + } catch (error) { + console.warn( + `Assertion tagging failed; continuing without assertion metadata: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return new Map(); + } +} diff --git a/worker/main.ts b/worker/main.ts index 591216b73..837800fa5 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -42,7 +42,8 @@ import { probeSupabaseHealth } from "../src/lib/supabase/health"; import type { Json, TablesInsert, TablesUpdate } from "../src/lib/supabase/database.types"; import type { ExtractedDocument, ImageEvidenceCategory } from "../src/lib/types"; import { buildAdditionalEmbeddingFieldInputs } from "./embedding-fields"; -import { checkPythonPdfPrerequisites } from "./prerequisites"; +import { checkMedspacyPrerequisites, checkPythonPdfPrerequisites } from "./prerequisites"; +import { annotateChunkAssertions, defaultAssertionTargets } from "./assertion-tagging"; import { buildTableFactRows } from "./table-facts"; type JobDocument = { @@ -1428,6 +1429,18 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { }, })), ); + if (env.WORKER_MEDSPACY_ASSERTION) { + // Fail-open (annotateChunkAssertions never throws): a broken medspaCy install + // degrades to unannotated chunks, never a failed ingestion job. + const assertions = await annotateChunkAssertions( + chunks.map((chunk, index) => ({ id: String(index), text: chunk.content })), + defaultAssertionTargets(), + ); + chunks.forEach((chunk, index) => { + const assertion = assertions.get(String(index)); + if (assertion) chunk.metadata = { ...chunk.metadata, assertion }; + }); + } const indexedChunkRows: IndexedChunkRow[] = []; for (let start = 0; start < chunks.length; start += env.WORKER_BATCH_SIZE) { @@ -1922,6 +1935,12 @@ async function main() { if (!prereqs.ok) { console.warn(`PDF/OCR prerequisite warning: ${prereqs.detail}`); } + if (env.WORKER_MEDSPACY_ASSERTION) { + const medspacyPrereqs = await checkMedspacyPrerequisites(); + if (!medspacyPrereqs.ok) { + console.warn(`medspaCy assertion prerequisite warning (tagging will fail open): ${medspacyPrereqs.detail}`); + } + } while (true) { let jobs: JobRow[] = []; diff --git a/worker/prerequisites.ts b/worker/prerequisites.ts index ab8bf0c19..70c247ce8 100644 --- a/worker/prerequisites.ts +++ b/worker/prerequisites.ts @@ -66,6 +66,27 @@ export function checkPythonPdfPrerequisites(): Promise { "print(json.dumps(result))", ].join("\n"); + return probePythonJson(script, "Python PDF/OCR prerequisites ready."); +} + +// Only enforced when WORKER_MEDSPACY_ASSERTION=true — medspacy is an optional +// dependency and workers without it must keep starting cleanly while the flag is off. +export function checkMedspacyPrerequisites(): Promise { + const script = [ + "import json", + "result = {'ok': True, 'missing': []}", + "try:", + " import medspacy", + "except Exception as exc:", + " result['ok'] = False", + " result['missing'].append(f'medspacy: {exc}')", + "print(json.dumps(result))", + ].join("\n"); + + return probePythonJson(script, "medspaCy assertion-tagging prerequisites ready."); +} + +function probePythonJson(script: string, readyDetail: string): Promise { return new Promise((resolve) => { const child = spawn(env.PYTHON_BIN, ["-c", script], { stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; @@ -89,7 +110,7 @@ export function checkPythonPdfPrerequisites(): Promise { const parsed = JSON.parse(stdout) as { ok: boolean; missing?: string[] }; resolve({ ok: parsed.ok, - detail: parsed.ok ? "Python PDF/OCR prerequisites ready." : `Missing ${parsed.missing?.join("; ")}`, + detail: parsed.ok ? readyDetail : `Missing ${parsed.missing?.join("; ")}`, }); } catch { resolve({ ok: false, detail: "Python prerequisite check returned invalid output." }); diff --git a/worker/python/analyze_assertions.py b/worker/python/analyze_assertions.py new file mode 100644 index 000000000..59a07e088 --- /dev/null +++ b/worker/python/analyze_assertions.py @@ -0,0 +1,127 @@ +"""Tag clinical assertion status for target terms inside chunk texts. + +Uses medspaCy's ConText implementation (negation / uncertainty / family / +historical modifiers). ConText only marks target entities, so the caller must +supply the target term list — the worker passes the clinical vocabulary, the +eval harness passes per-fixture targets. + +Usage: analyze_assertions.py input.json [output.json] + +Input JSON: {"chunks": [{"id": str, "text": str}], "targets": [str, ...]} +Output JSON: {"assertions": [{"id": str, "negated_terms": [str], "uncertain_terms": [str], + "family_terms": [str], "historical_terms": [str]}], + "version": str, "warnings": [str]} + +Exit codes mirror extract_pdf_assets.py: 1 usage error, 2 missing dependency. +""" + +import json +import sys + +try: + import medspacy + from medspacy.context import ConTextRule + from medspacy.target_matcher import TargetRule +except Exception as exc: # noqa: BLE001 - report any import failure and exit + sys.stderr.write(f"medspacy unavailable: {exc}\n") + sys.exit(2) + +# AU clinical phrasing missing from medspaCy's default (US-leaning) ConText rules. +# Gaps were identified by scripts/fixtures/assertion-golden.json via +# `npm run eval:assertions` — extend the fixture first when adding rules here. +AU_CONTEXT_RULES = [ + ConTextRule("nil", "NEGATED_EXISTENCE", direction="FORWARD"), + # max_scope bounds the forward reach so a cue over one finding does not leak onto + # later mentions in the same sentence ("Suspected myocarditis ... of clozapine"). + ConTextRule("query", "POSSIBLE_EXISTENCE", direction="FORWARD", max_scope=4), + ConTextRule("suspected", "POSSIBLE_EXISTENCE", direction="FORWARD", max_scope=4), + ConTextRule("may represent", "POSSIBLE_EXISTENCE", direction="FORWARD", max_scope=4), + ConTextRule("may have", "POSSIBLE_EXISTENCE", direction="FORWARD", max_scope=4), + ConTextRule("differential diagnosis includes", "POSSIBLE_EXISTENCE", direction="FORWARD"), + ConTextRule("impression:", "POSSIBLE_EXISTENCE", direction="FORWARD"), + ConTextRule("previous episode of", "HISTORICAL", direction="FORWARD"), + ConTextRule("previous episodes of", "HISTORICAL", direction="FORWARD"), + # A semicolon starts a new clause; modifier scope must not leak across it + # ("Possible NMS; cease antipsychotic" must not mark the antipsychotic uncertain). + ConTextRule(";", "TERMINATE", direction="TERMINATE"), +] + + +def build_nlp(targets): + nlp = medspacy.load() + matcher = nlp.get_pipe("medspacy_target_matcher") + matcher.add([TargetRule(literal=term, category="CLINICAL_TERM") for term in targets]) + nlp.get_pipe("medspacy_context").add(AU_CONTEXT_RULES) + return nlp + + +def analyze_chunk(nlp, chunk): + doc = nlp(chunk["text"]) + result = { + "id": chunk["id"], + "negated_terms": [], + "uncertain_terms": [], + "family_terms": [], + "historical_terms": [], + } + seen = set() + for ent in doc.ents: + term = ent.text.lower() + flags = ( + ("negated_terms", bool(ent._.is_negated)), + ("uncertain_terms", bool(ent._.is_uncertain) or bool(ent._.is_hypothetical)), + ("family_terms", bool(ent._.is_family)), + ("historical_terms", bool(ent._.is_historical)), + ) + for key, flagged in flags: + if flagged and (key, term) not in seen: + seen.add((key, term)) + result[key].append(term) + return result + + +def run(input_path, output_path=None): + with open(input_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) + + chunks = payload.get("chunks") or [] + targets = [term for term in (payload.get("targets") or []) if isinstance(term, str) and term.strip()] + warnings = [] + if not targets: + warnings.append("no targets supplied; nothing to tag") + + nlp = build_nlp(targets) if targets else None + assertions = [] + for chunk in chunks: + if not isinstance(chunk, dict) or not isinstance(chunk.get("text"), str) or "id" not in chunk: + warnings.append(f"skipped malformed chunk: {chunk!r:.120}") + continue + if nlp is None: + assertions.append( + { + "id": chunk["id"], + "negated_terms": [], + "uncertain_terms": [], + "family_terms": [], + "historical_terms": [], + } + ) + continue + assertions.append(analyze_chunk(nlp, chunk)) + + result = { + "assertions": assertions, + "version": getattr(medspacy, "__version__", "unknown"), + "warnings": warnings, + } + if output_path: + with open(output_path, "w", encoding="utf-8") as handle: + json.dump(result, handle) + print(json.dumps(result)) + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3): + sys.stderr.write("usage: analyze_assertions.py input.json [output.json]\n") + sys.exit(1) + run(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else None) diff --git a/worker/python/requirements.txt b/worker/python/requirements.txt index ef68caaa4..d0734e1b1 100644 --- a/worker/python/requirements.txt +++ b/worker/python/requirements.txt @@ -1,3 +1,6 @@ PyMuPDF>=1.24.0 pillow>=10.0.0 pytesseract>=0.3.10 +# Optional: clinical assertion tagging (negation/uncertainty), used only when +# WORKER_MEDSPACY_ASSERTION=true. medspaCy pins its own compatible spaCy version. +medspacy>=1.1.0