diff --git a/docs/process-hardening.md b/docs/process-hardening.md index d9b7ff927..134e9a2ab 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -202,3 +202,9 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went - **Behaviour:** inside an Answer thread, browser Back changes the URL (`/?mode=answer&q=A&run=1` ← `...q=B&run=1`) but the rendered answer/thread does not change. Two guards produce this: the auto-run effect skips when `run=1` is already present, and the answer view early-returns when an answer is already on screen (`ClinicalDashboard`). This is thread persistence by design, not an accident — clearing the thread on Back would destroy in-progress clinical context. - **Open product decision:** either (a) accept that the URL is not a faithful pointer to the visible answer inside a thread (current state), or (b) make Back restore the previous question's answer from thread history. Option (b) needs answer-thread state keyed by URL and re-render on `popstate`/param change; it is a deliberate feature, not a bug fix. - **Guardrail until decided:** do not "fix" the skipped auto-run on Back in isolation — re-running the search on Back would clobber the visible thread with a regenerated (and possibly different) answer, which is worse than either deliberate option. + +## Audit re-triage & property-test suite (2026-07-06) + +- **Full re-triage of the 2026-07-01 audit's H1–H4 and M1–M17 against current main: all 21 are closed.** The 2026-07-02 remediation branch landed (H1/H2/H4+M8+M16, M1–M12, M14, M15, M17 all carry audit-referencing fixes verified in code, not just comments); PR #278 added the P0 carry-over (including the M9 DELETE TOCTOU re-check); M13's migration (`20260702000000`) is in-tree (live-apply verification needs `npm run check:m13-migration` with live keys). H3 is closed by supersession: PR #118 removed the governance penalties it complained about entirely (measured, "do not reintroduce" comment in `retrieval-selection.ts`); the residual `Math.max` floor only affects intent boosts, and un-flooring it was already tried and withdrawn (compounding across selection passes — audit R1). Do not re-open H3 without the golden retrieval eval. +- **New fast-check property suite** (`tests/property-*.test.ts`) pins the text-processing core's clinical invariants over generated inputs: unit-bearing numeric tokens survive sanitization; chunking terminates (including `CHUNK_OVERLAP >= CHUNK_SIZE`, M17) and neither loses nor invents content; table normalization stays rectangular, preserves numeric values, and the low-confidence caveat survives both clipboard export paths (H4). The suite immediately caught a live line-level H2 residue in `stripLowYieldLines` (control markers glued to threshold sentences deleted the whole line), fixed in the same PR. +- **Eval debt (blocking merge of the sanitizer fix, not development):** the environment had no live Supabase/OpenAI keys, so `npm run eval:quality -- --rag-only` (answer-path gate for the sanitizer change) could not run. Run it before merging `claude/audit-sweep-property-tests`. No retrieval-selection/ranking files were touched, so the golden retrieval eval is not required for this branch. diff --git a/package-lock.json b/package-lock.json index 594ea5d86..64a8061f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.39.4", "eslint-config-next": "16.2.10", + "fast-check": "^4.8.0", "playwright": "^1.61.1", "prettier": "^3.9.4", "tailwindcss": "^4.3.1", @@ -5428,6 +5429,29 @@ "node": ">=12.0.0" } }, + "node_modules/fast-check": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.8.0.tgz", + "integrity": "sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-csv": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", @@ -8252,6 +8276,23 @@ "node": ">=6" } }, + "node_modules/pure-rand": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.1.tgz", + "integrity": "sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/package.json b/package.json index 03dff5c84..c690f8b7f 100644 --- a/package.json +++ b/package.json @@ -134,6 +134,7 @@ "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.39.4", "eslint-config-next": "16.2.10", + "fast-check": "^4.8.0", "playwright": "^1.61.1", "prettier": "^3.9.4", "tailwindcss": "^4.3.1", diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index 1bcef91a9..4d05c51ff 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -219,7 +219,15 @@ function stripLowYieldLines(value: string) { const isControlLine = sourceControlLinePattern.test(normalized); const hasSourceMarker = sourceDocumentCodeTestPattern.test(normalized) || pageBoilerplateTestPattern.test(normalized); - const hasClinicalSignal = clinicalSignalPattern.test(normalized); + // H2 (line-level): extraction glues control markers ("Document owner:", + // "review date") onto body text, and several callers compact the whole + // excerpt to a single line before this filter runs — so dropping the + // line deletes clinical content along with the marker. A line carrying + // threshold-bearing values ("8 or below", "3 to 15") must survive even + // when it lacks a clinical keyword, same generous bias as the + // fragment-level rescue below. + const hasClinicalSignal = + clinicalSignalPattern.test(normalized) || clinicalThresholdSignalPattern.test(normalized); if (isControlLine && !hasClinicalSignal) return false; if (hasSourceMarker && normalized.length <= 140 && !hasClinicalSignal) return false; return true; diff --git a/tests/property-accessible-table.test.ts b/tests/property-accessible-table.test.ts new file mode 100644 index 000000000..a6ff21482 --- /dev/null +++ b/tests/property-accessible-table.test.ts @@ -0,0 +1,206 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { normalizeAccessibleTable } from "../src/lib/accessible-table-normalization"; +import { formatAnswerForClipboard, formatWardNote } from "../src/lib/ward-output"; +import type { RagAnswer } from "../src/lib/types"; + +// Properties for clinical table normalization and its export paths. +// +// The 2026-07-01 audit's H4/M8/M16 cluster was about copied ward-note tables +// silently diverging from the on-screen table: dropped low-confidence caveats, +// duplicated header rows, and ragged rows shifting values under the wrong +// column. These properties pin the invariants over generated tables: +// 1. normalization always yields a rectangular grid, +// 2. no numeric value is invented or lost by normalization, +// 3. rows are only ever dropped/merged with the lowConfidence flag raised +// (for clinical tables), and +// 4. the lowConfidence caveat survives BOTH clipboard export paths. + +const LOW_CONFIDENCE_CAVEAT = "verify values against the source document"; + +const namedHeaders = ["ANC level", "Action", "Dose", "Frequency", "Parameter", "Escalation"] as const; +const cellPool = [ + "Below 1.5", + "1.5-2.0", + "Above 2.0", + "12.5 mg daily", + "withhold dose", + "increase monitoring", + "contact prescriber", + "review weekly", + "continue treatment", + "", +] as const; +const nonEmptyCellPool = cellPool.filter(Boolean); + +const cell = fc.constantFrom(...cellPool); +const nonEmptyCell = fc.constantFrom(...nonEmptyCellPool); + +function distinctHeaders(count: number) { + return fc.shuffledSubarray([...namedHeaders], { minLength: count, maxLength: count }).map((headers) => [...headers]); +} + +// Ragged rows: cell counts deliberately range past the header width (M16). +function rowsFor(columnCount: number) { + return fc.array( + fc + .tuple(nonEmptyCell, fc.array(cell, { minLength: 0, maxLength: columnCount })) + .map(([first, rest]) => [first, ...rest]), + { minLength: 2, maxLength: 6 }, + ); +} + +const cleanTable = fc + .integer({ min: 2, max: 4 }) + .chain((columnCount) => fc.tuple(distinctHeaders(columnCount), rowsFor(columnCount))); + +// A clinical table with a generic column interleaved between named ones — the +// ambiguity that must force the conservative raw-grid fallback + caveat. +const ambiguousClinicalTable = fc + .tuple(distinctHeaders(2), rowsFor(3)) + .map(([headers, rows]) => ({ columns: ["ANC level", "", headers[1]], rows })); + +function numericTokens(cells: string[]) { + return new Set(cells.join(" ").match(/\d+(?:\.\d+)?/g) ?? []); +} + +function tableCells(table: { header: string[]; body: string[][] }) { + return [...table.header, ...table.body.flat()]; +} + +function answerWithTable(rows: string[][], columns: string[]): RagAnswer { + // Mirrors the H4 regression fixture in ward-output.test.ts: a grounded + // answer whose visual evidence carries threshold rows, so the table is + // promoted into the thresholds section of both export formats. + return { + answer: "Withhold clozapine if ANC is below the required threshold and urgently review.", + grounded: true, + confidence: "medium", + citations: [ + { + chunk_id: "chunk-1", + document_id: "doc-1", + title: "Clozapine source", + file_name: "clozapine.pdf", + page_number: 2, + chunk_index: 0, + }, + ], + sources: [], + answerSections: [ + { + heading: "Threshold", + body: "Withhold clozapine if ANC is below the required threshold and urgently review.", + citation_chunk_ids: ["chunk-1"], + }, + ], + visualEvidence: [ + { + id: "image-1", + image_id: "image-1", + signed_url_endpoint: "/api/images/image-1/signed-url", + caption: "FBC/ANC monitoring thresholds", + document_id: "doc-1", + title: "Clozapine source", + file_name: "clozapine.pdf", + page_number: 2, + source_chunk_id: "chunk-1", + chunk_index: 0, + viewer_href: "/documents/doc-1?page=2&chunk=chunk-1", + tableLabel: "Table 1", + tableTitle: "FBC/ANC thresholds", + tableRows: rows, + tableColumns: columns, + }, + ], + }; +} + +describe("property: accessible table normalization", () => { + it("always yields a rectangular grid with no invented or lost numeric values", () => { + fc.assert( + fc.property(cleanTable, ([columns, rows]) => { + const normalized = normalizeAccessibleTable(rows, columns, { conservativeClinical: true }); + if (!normalized) return; + + for (const row of normalized.body) { + expect(row).toHaveLength(normalized.header.length); + } + + const inputTokens = numericTokens([...columns, ...rows.flat()]); + const outputTokens = numericTokens(tableCells(normalized)); + for (const token of outputTokens) { + expect(inputTokens).toContain(token); + } + for (const token of inputTokens) { + expect(outputTokens).toContain(token); + } + }), + ); + }); + + it("preserves the row count exactly when every row anchors on a non-empty first cell", () => { + fc.assert( + fc.property(cleanTable, ([columns, rows]) => { + const normalized = normalizeAccessibleTable(rows, columns, { conservativeClinical: true }); + const expectedRows = rows.filter((row) => row.some((value) => value.trim() && !/^[-–—]+$/.test(value.trim()))); + expect(normalized?.body ?? []).toHaveLength(expectedRows.length); + }), + ); + }); + + it("flags ambiguous clinical tables low-confidence and preserves their raw row/column counts", () => { + fc.assert( + fc.property(ambiguousClinicalTable, ({ columns, rows }) => { + const normalized = normalizeAccessibleTable(rows, columns, { conservativeClinical: true }); + expect(normalized).not.toBeNull(); + expect(normalized?.lowConfidence).toBe(true); + + // Conservative fallback: raw grid preserved 1:1 — no merges, no drops. + const sourceColumnCount = Math.max(columns.length, ...rows.map((row) => row.length)); + expect(normalized?.header).toHaveLength(sourceColumnCount); + const nonEmptyRows = rows.filter((row) => row.some((value) => value.trim() && !/^[-–—]+$/.test(value.trim()))); + expect(normalized?.body).toHaveLength(nonEmptyRows.length); + }), + ); + }); +}); + +describe("property: low-confidence caveat survives every export path (H4)", () => { + it("ward note and clipboard always carry the caveat when normalization is low-confidence", () => { + fc.assert( + fc.property(ambiguousClinicalTable, ({ columns, rows }) => { + const answer = answerWithTable(rows, columns); + for (const exported of [formatAnswerForClipboard(answer), formatWardNote(answer, false)]) { + expect(exported).toContain(LOW_CONFIDENCE_CAVEAT); + } + }), + ); + }); + + it("clean tables do not gain a spurious caveat, and low-confidence ones render their rows (canary)", () => { + fc.assert( + fc.property(cleanTable, ([columns, rows]) => { + const normalized = normalizeAccessibleTable(rows, columns, { conservativeClinical: true }); + fc.pre(Boolean(normalized) && !normalized?.lowConfidence); + const exported = formatAnswerForClipboard(answerWithTable(rows, columns)); + expect(exported).not.toContain(LOW_CONFIDENCE_CAVEAT); + }), + ); + + // Non-vacuity canary: the ambiguous fixture really is promoted into the + // exported note — the caveat property above cannot pass on an empty + // output. Uses the known-ambiguous shape from the H4 regression test. + const exported = formatAnswerForClipboard( + answerWithTable( + [ + ["Below 1.5", "withhold dose", "contact prescriber"], + ["1.5-2.0", "increase monitoring", "review threshold"], + ], + ["ANC level", "", "Action"], + ), + ); + expect(exported).toContain("| Below 1.5 |"); + expect(exported).toContain(LOW_CONFIDENCE_CAVEAT); + }); +}); diff --git a/tests/property-chunking.test.ts b/tests/property-chunking.test.ts new file mode 100644 index 000000000..a0547e4d6 --- /dev/null +++ b/tests/property-chunking.test.ts @@ -0,0 +1,124 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { chunkTextWithOverlap } from "../src/lib/chunking"; + +// Properties: chunking always terminates and never loses or invents content. +// +// The 2026-07-01 audit found chunkTextBySentence could spin forever when +// CHUNK_OVERLAP >= CHUNK_SIZE (M17) — both values pass env validation +// independently, so the combination is reachable by configuration alone. The +// generators here include that region on purpose: completion of every run IS +// the termination proof (vitest's 15s test timeout is the backstop). +// +// The generated text avoids line shapes removePageNoise is contracted to drop +// (standalone page footers, bare page numbers, boilerplate) so the coverage +// assertions target the chunker itself, not the noise filter: every paragraph +// starts with an alphabetic word and no vocabulary word can form a +// noise-pattern line. + +const vocabulary = [ + "monitor", + "serum", + "level", + "renal", + "function", + "clozapine", + "lithium", + "review", + "daily", + "weekly", + "titrate", + "dose", + "before", + "after", + "the", + "and", + "with", + "until", + "stable", + "baseline", + "thyroid", + "calcium", + "escalate", + "urgent", + "assessment", + "12.5", + "mg", + "0.8-1.2", + "mmol", +] as const; + +const word = fc.constantFrom(...vocabulary); +// removePageNoise treats whole lines of <= 2 characters as extraction debris +// (looksLikeMetadataNoise), so every paragraph anchors on a word long enough +// to keep its line alive; the anchor is alphabetic so no line can match the +// bare-number or page-footer noise patterns either. (One contract exception: +// a unit-only line like "mg" directly after a digit-ending line is rejoined +// to that line rather than dropped — rejoinWrappedDoseUnits in chunking.ts. +// The generators never emit that shape: unit tokens only appear space-joined +// inside a paragraph line, never alone on their own line.) +const anchorWord = fc.constantFrom(...vocabulary.filter((entry) => /^[a-z]{3,}/.test(entry))); + +const paragraph = fc + .tuple(anchorWord, fc.array(word, { minLength: 0, maxLength: 120 })) + .map(([first, rest]) => [first, ...rest].join(" ")); + +const documentText = fc.array(paragraph, { minLength: 1, maxLength: 6 }).map((paragraphs) => paragraphs.join("\n\n")); + +function strippedChars(value: string) { + return value.replace(/\s+/g, ""); +} + +// True when `needle`'s characters appear in `haystack` in order (gaps allowed). +// Overlapping chunks duplicate content in the haystack, which a plain equality +// check would reject; a subsequence check accepts duplication but still fails +// on any dropped or reordered character. +function isSubsequence(needle: string, haystack: string) { + let position = 0; + for (const char of haystack) { + if (char === needle[position]) position += 1; + if (position === needle.length) return true; + } + return position === needle.length; +} + +function normalizeWhitespace(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +describe("property: chunking terminates and covers its input", () => { + it("terminates and preserves every character for any size/overlap combination, including overlap >= chunkSize (M17)", () => { + fc.assert( + fc.property( + documentText, + fc.integer({ min: 40, max: 400 }), + fc.integer({ min: 0, max: 500 }), + (text, chunkSize, overlap) => { + const chunks = chunkTextWithOverlap(text, chunkSize, overlap); + + // Coverage: nothing the cleaner kept may be lost. The generated text + // contains no removable noise, so the input's own character stream + // (whitespace aside) must survive into the chunks, in order. + expect(isSubsequence(strippedChars(text), strippedChars(chunks.join("")))).toBe(true); + + // No invention or reordering: every chunk reads as a contiguous run + // of the input. (Overlap tails are suffixes of the preceding + // paragraph, so they remain contiguous in the normalized input.) + const normalizedInput = normalizeWhitespace(text); + for (const chunk of chunks) { + expect(normalizedInput).toContain(normalizeWhitespace(chunk)); + } + }, + ), + ); + }); + + it("returns the whole cleaned text as a single chunk when it fits", () => { + fc.assert( + fc.property(paragraph, (text) => { + const chunks = chunkTextWithOverlap(text, text.length + 10, 20); + expect(chunks).toEqual([text]); + }), + ); + }); +}); diff --git a/tests/property-numeric-token-preservation.test.ts b/tests/property-numeric-token-preservation.test.ts new file mode 100644 index 000000000..af645922b --- /dev/null +++ b/tests/property-numeric-token-preservation.test.ts @@ -0,0 +1,132 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { + clinicalProseUsefulness, + sourceTextForDisplay, + sourceTextForModel, + sourceTextForVerbatimQuote, +} from "../src/lib/source-text-sanitizer"; + +// Property: no numeric token with a unit ever disappears through sanitization. +// +// The sanitizer is allowed to drop provenance noise, banners, title fragments, +// and boilerplate — but a dose/threshold figure embedded in clinical prose is +// exactly the content whose silent loss the 2026-07-01 audit flagged as a +// clinical-safety failure (H2: the GCS threshold sentence). These properties +// pin that guarantee over a generated input space instead of single examples. +// +// The generated prose deliberately avoids tokens the sanitizer is CONTRACTED +// to remove (document codes, "Page N of N", "OFFICIAL" banners, "… evidence:" +// labels, chunk/similarity telemetry) so the assertions target preservation of +// clinical values, not the noise-stripping behaviour itself. Noise of those +// shapes is injected AROUND the dose sentences and carries no assertions. + +const drugs = ["clozapine", "lithium", "olanzapine", "quetiapine", "sertraline", "haloperidol"] as const; + +// Units recognized by the sanitizer's threshold-rescue pattern +// (clinicalThresholdSignalPattern) and by answer-verification's extractor. +const units = ["mg", "mcg", "micrograms", "mmol", "units", "hours", "mmHg", "bpm"] as const; + +const integerValue = fc.integer({ min: 1, max: 2000 }).map(String); +const decimalValue = fc + .tuple(fc.integer({ min: 0, max: 40 }), fc.integer({ min: 1, max: 9 })) + .map(([whole, tenth]) => `${whole}.${tenth}`); +const numericValue = fc.oneof(integerValue, decimalValue); + +// A unit-bearing dose/threshold token, in the spellings that appear in real +// extracted source text: spaced ("12.5 mg"), attached ("100mg"), ranges +// ("25-50 mg"), percentages ("1.85%"), and comparatives ("8 or below"). +const doseToken = fc.oneof( + fc.tuple(numericValue, fc.constantFrom(...units)).map(([value, unit]) => `${value} ${unit}`), + fc.tuple(numericValue, fc.constantFrom("mg", "mcg")).map(([value, unit]) => `${value}${unit}`), + fc.tuple(integerValue, integerValue, fc.constantFrom(...units)).map(([low, high, unit]) => `${low}-${high} ${unit}`), + decimalValue.map((value) => `${value}%`), + integerValue.map((value) => `${value} or below`), +); + +// Sentences that carry the dose token. Every template yields >= 3 tokens (the +// sanitizer unconditionally drops shorter fragments) and none starts with the +// "the retrieved/supplied/provided/indexed" meta-intro the sanitizer removes. +const doseSentence = fc + .tuple(doseToken, fc.constantFrom(...drugs), fc.integer({ min: 0, max: 4 })) + .map(([token, drug, template]) => { + switch (template) { + case 0: + return { token, sentence: `Administer ${drug} ${token} daily with food.` }; + case 1: + return { token, sentence: `Withhold ${drug} when the count is ${token} and contact the prescriber.` }; + case 2: + return { token, sentence: `Titrate ${drug} to ${token} over three days.` }; + case 3: + // Title-shaped threshold sentence — the audit-H2 GCS case: a sentence + // that looks like a source-title fragment ("… Scale …") but carries + // clinical values must never be dropped. + return { + token, + sentence: `The Glasgow Coma Scale ranges from 3 to 15 with ${token} indicating severe head injury.`, + }; + default: + return { token, sentence: `Monitor the serum level and maintain ${token} until review.` }; + } + }); + +// Droppable noise the sanitizer exists to remove. No assertions attach to it. +const noiseSentence = fc.constantFrom( + "Neuroleptic side effect Guideline Appendix 1.", + "OFFICIAL", + "Page 3 of 5", + "Document owner: Pharmacy Department.", + "Uncontrolled when printed.", + "Mental Health Procedure PAE-PRO-0338/16.", + "LUNSERS (Liverpool University Neuroleptic Side Effect Rating Scale) rating scale appendix.", +); + +const separator = fc.constantFrom(" ", "\n", "\n\n"); + +const clinicalDocument = fc + .tuple( + fc.array(doseSentence, { minLength: 1, maxLength: 4 }), + fc.array(noiseSentence, { minLength: 0, maxLength: 4 }), + separator, + fc.boolean(), + ) + .map(([doses, noise, join, noiseFirst]) => { + const parts = noiseFirst + ? [...noise, ...doses.map((dose) => dose.sentence)] + : [...doses.map((dose) => dose.sentence), ...noise]; + return { text: parts.join(join), tokens: doses.map((dose) => dose.token) }; + }); + +function normalizeWhitespace(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +describe("property: unit-bearing numeric tokens survive sanitization", () => { + it("clinicalProseUsefulness never drops a fragment carrying a dose/threshold token", () => { + fc.assert( + fc.property(clinicalDocument, ({ text, tokens }) => { + const kept = normalizeWhitespace(clinicalProseUsefulness(text).text); + for (const token of tokens) { + expect(kept).toContain(normalizeWhitespace(token)); + } + }), + ); + }); + + it("display, model, and verbatim-quote sanitizers preserve dose/threshold tokens", () => { + fc.assert( + fc.property(clinicalDocument, ({ text, tokens }) => { + for (const sanitized of [ + sourceTextForDisplay(text), + sourceTextForModel(text), + sourceTextForVerbatimQuote(text), + ]) { + const haystack = normalizeWhitespace(sanitized); + for (const token of tokens) { + expect(haystack).toContain(normalizeWhitespace(token)); + } + } + }), + ); + }); +}); diff --git a/tests/source-text-sanitizer.test.ts b/tests/source-text-sanitizer.test.ts index c875af3f9..4ce1de442 100644 --- a/tests/source-text-sanitizer.test.ts +++ b/tests/source-text-sanitizer.test.ts @@ -130,6 +130,26 @@ describe("source text sanitizer", () => { expect(usefulness.text).toContain("Assess the patient on admission"); }); + // H2 line-level residue (found by the fast-check property suite, + // 2026-07-06): extraction glues document-control markers onto body text and + // stripInternalImageDataBlocks compacts the excerpt to a single line, so the + // control-line filter in stripLowYieldLines deleted the whole line — GCS + // thresholds included — when the sentence carried values but no clinical + // keyword. + it("keeps threshold-bearing lines glued to document-control markers (H2 line-level)", () => { + const text = + "The Glasgow Coma Scale ranges from 3 to 15 with 8 or below indicating severe head injury. Document owner: Pharmacy Department."; + + expect(sourceTextForDisplay(text)).toContain("8 or below"); + expect(clinicalProseUsefulness(text).text).toContain("ranges from 3 to 15"); + }); + + it("still drops pure document-control lines without clinical values", () => { + const text = "Document owner: Pharmacy Department.\nUncontrolled when printed."; + + expect(sourceTextForDisplay(text)).toBe(""); + }); + it("still drops bare-integer title noise like 'Guideline Appendix 1' (H2 guard)", () => { const noisy = "Dose evidence: LUNSERS (Liverpool University Neuroleptic Side Effect Rating Scale) - using for monitoring Neuroleptic side effect Guideline Appendix 1.";