From 81ab9696da4b330ca0b2e5519891a9942f90421b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 16:53:54 +0000 Subject: [PATCH 1/2] feat(eval): emit canary retrieval artifact + alias-aware ranking-snapshot builder with freshness provenance - eval-canary.yml golden step writes --json-out and uploads .local/eval-canary/ (pinned upload-artifact, 30-day retention) so ranking-snapshot regeneration no longer needs a paid dispatch - --json-out no longer forces --json: the artifact file writes independently while stdout keeps the human-readable log the failure-issue analyzer parses - clinicalDocumentAliases/clinicalContentAliases move verbatim to scripts/lib/clinical-aliases.ts, shared by the live gates and the snapshot builder; the builder now grades documentMatch/contentMatch through the sanctioned aliases so tuner ground truth agrees with the live eval - snapshot metadata gains generatedAt + optional sourceRunId provenance; validateRankingSnapshot accepts them and relaxes exactly-36 to at-least-36 for the planned fixture expansion; a 30-day freshness test activates on first regeneration - build:ranking-snapshot npm script for discoverability Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- .github/workflows/eval-canary.yml | 27 ++++++++- package.json | 1 + scripts/build-ranking-snapshot.ts | 37 ++++++++++--- scripts/eval-retrieval.ts | 62 ++++----------------- scripts/lib/clinical-aliases.ts | 72 ++++++++++++++++++++++++ scripts/lib/ranking-snapshot-builder.ts | 24 ++++++++ scripts/lib/ranking-tuning.ts | 18 +++++- tests/ranking-tuning.test.ts | 74 +++++++++++++++++++++++-- 8 files changed, 247 insertions(+), 68 deletions(-) create mode 100644 scripts/lib/clinical-aliases.ts diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index 2f06cac76..4336ebdde 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -3,7 +3,8 @@ # Runs the golden retrieval eval (which PR CI can never run — it needs live # Supabase + OpenAI keys) plus a small answer-quality subset against the live # project, and fails loudly on regression: red run + a GitHub issue on -# scheduled failures. +# scheduled failures. Each run also uploads the eval JSON artifact that feeds +# offline ranking-snapshot regeneration (scripts/build-ranking-snapshot.ts). # # The schedule only fires from the default branch. After merging, trigger one # workflow_dispatch run and confirm it is green before trusting the weekly @@ -97,7 +98,10 @@ jobs: run: | mkdir -p .local/eval-canary set -o pipefail - npm run eval:retrieval:quality -- --fail-on-threshold 2>&1 | tee .local/eval-canary/golden-retrieval.log + # --json-out writes the per-case artifact (input for scripts/build-ranking-snapshot.ts) + # without changing stdout — the tee'd log keeps the human-readable per-case lines the + # failure-issue analyzer parses. + npm run eval:retrieval:quality -- --fail-on-threshold --json-out .local/eval-canary/golden-retrieval.json 2>&1 | tee .local/eval-canary/golden-retrieval.log - name: Answer-quality subset (live generation) id: answer_quality @@ -115,6 +119,25 @@ jobs: set -o pipefail npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log + # Uploaded on success AND failure: the JSON artifact regenerates the offline ranking + # snapshot (download it, then `npm run build:ranking-snapshot -- --input --output + # scripts/fixtures/rag-ranking-candidate-snapshot.v1.json --source-run-id `), and + # the logs carry failure diagnostics. Contents match what the tee'd step logs already + # expose (titles/file names, score telemetry, 220-char previews of the all-public corpus) + # — no env values or secrets are written to .local/eval-canary. + - name: Upload eval output artifact + if: always() && steps.install.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: eval-canary-output + path: .local/eval-canary/ + # .local/ is a hidden directory (the repo's gitignored scratch convention), which + # upload-artifact excludes by default. + include-hidden-files: true + # Matches the snapshot freshness window enforced by tests/ranking-tuning.test.ts. + retention-days: 30 + if-no-files-found: ignore + - name: Open or update canary failure issue if: failure() && github.event_name == 'schedule' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/package.json b/package.json index 6ff69ab81..b41423270 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,7 @@ "profile:retrieval": "node scripts/run-tsx.mjs scripts/profile-retrieval-rpcs.ts", "warm:retrieval": "node scripts/run-tsx.mjs scripts/warm-retrieval-cache.ts", "tune:search-weights": "node scripts/run-tsx.mjs scripts/tune-search-weights.ts", + "build:ranking-snapshot": "node scripts/run-tsx.mjs scripts/build-ranking-snapshot.ts", "cleanup:storage": "node scripts/run-tsx.mjs scripts/cleanup-storage.ts", "purge:query-logs": "node scripts/run-tsx.mjs scripts/purge-query-logs.ts", "audit:tables": "node scripts/run-tsx.mjs scripts/audit-tables.ts", diff --git a/scripts/build-ranking-snapshot.ts b/scripts/build-ranking-snapshot.ts index 0672f0f0c..10814f60e 100644 --- a/scripts/build-ranking-snapshot.ts +++ b/scripts/build-ranking-snapshot.ts @@ -2,7 +2,12 @@ import { createHash } from "node:crypto"; import { readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import type { RagQueryClass } from "../src/lib/types"; -import { candidateFeatures, labelMatches, type ArtifactCandidate } from "./lib/ranking-snapshot-builder"; +import { + candidateFeatures, + contentLabelMatchesWithAliases, + documentLabelMatchesWithAliases, + type ArtifactCandidate, +} from "./lib/ranking-snapshot-builder"; import { RANKING_SNAPSHOT_VERSION, type RankingCandidateFeatures, @@ -73,9 +78,15 @@ const hardNegativeTemplates: Array<{ features: RankingCandidateFeatures; }>; -function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot { - if (!Array.isArray(artifact.results) || artifact.results.length !== 36) { - throw new Error("Expected a retrieval artifact containing exactly 36 cases"); +function convertArtifact(artifact: RetrievalArtifact, sourceRunId?: string): RankingSnapshot { + // Floor, not an exact pin: the golden fixture only ever grows, and a short artifact means a + // truncated or filtered eval run that must not silently become the tuner's ground truth. + if (!Array.isArray(artifact.results) || artifact.results.length < 36) { + throw new Error( + `Expected a retrieval artifact containing at least 36 cases (got ${ + Array.isArray(artifact.results) ? artifact.results.length : 0 + })`, + ); } const cases = artifact.results.map((testCase) => { const expectedDocuments = testCase.expectedDocumentSubstrings ?? []; @@ -83,9 +94,10 @@ function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot { const candidates = (testCase.topResults ?? []).map((candidate, index): RankingSnapshotCandidate => { const documentText = `${candidate.title ?? ""} ${candidate.file_name ?? ""}`.toLowerCase(); const contentText = (candidate.content_preview ?? "").toLowerCase(); - const documentMatch = expectedDocuments.some((label) => labelMatches(documentText, label)); + const documentMatch = expectedDocuments.some((label) => documentLabelMatchesWithAliases(documentText, label)); const contentMatch = - expectedContent.length > 0 && expectedContent.every((label) => labelMatches(contentText, label)); + expectedContent.length > 0 && + expectedContent.every((label) => contentLabelMatchesWithAliases(contentText, label)); return { candidateHash: candidateHash(`${testCase.id}:${candidate.chunk_id ?? `rank-${index + 1}`}`), relevanceGrade: documentMatch && contentMatch ? 3 : documentMatch ? 2 : contentMatch ? 1 : 0, @@ -116,6 +128,10 @@ function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot { schema: "rag-ranking-candidate-snapshot", version: RANKING_SNAPSHOT_VERSION, sourceCaseCount: cases.length, + // Freshness provenance: tests/ranking-tuning.test.ts fails the build when generatedAt is + // older than 30 days, so a checked-in snapshot cannot silently drift from the live corpus. + generatedAt: new Date().toISOString(), + ...(sourceRunId ? { sourceRunId } : {}), sanitization: { candidateIdentity: "sha256", excludes: ["raw_uuid", "source_passage", "patient_data", "provider_metadata", "document_storage_path"], @@ -127,10 +143,15 @@ function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot { function main() { const input = argument("--input"); const output = argument("--output"); + // Optional provenance: the GitHub Actions run id of the eval-canary run whose artifact + // produced --input, so a snapshot can be traced back to its live eval. + const sourceRunId = argument("--source-run-id"); if (!input || !output) - throw new Error("Usage: build-ranking-snapshot --input --output "); + throw new Error( + "Usage: build-ranking-snapshot --input --output [--source-run-id ]", + ); const artifact = JSON.parse(readFileSync(resolve(input), "utf8")) as RetrievalArtifact; - const snapshot = validateRankingSnapshot(convertArtifact(artifact)); + const snapshot = validateRankingSnapshot(convertArtifact(artifact, sourceRunId)); writeFileSync(resolve(output), `${JSON.stringify(snapshot, null, 2)}\n`, "utf8"); console.log(JSON.stringify({ output: resolve(output), cases: snapshot.cases.length, version: snapshot.version })); } diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 897450f4b..1ea4f96d3 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -12,6 +12,9 @@ import { resolveEvalOwnerId, withProviderBackoff, } from "./eval-utils"; +// Alias tables live in a shared module so the offline ranking-snapshot builder grades +// candidates with the same sanctioned expansions this live eval's gates use. +import { clinicalContentAliases, clinicalDocumentAliases } from "./lib/clinical-aliases"; loadEnvConfig(process.cwd()); @@ -173,10 +176,10 @@ function parseArgs(argv: string[]): EvalArgs { if (token === "--owner-id") args.ownerId = value; if (token === "--limit") args.limit = Number.parseInt(value, 10); if (token === "--query") args.query = value; - if (token === "--json-out") { - args.jsonOut = value; - args.json = true; - } + // Deliberately independent of --json: the eval canary tees stdout into a log that both + // humans and the failure-issue analyzer read, so the artifact file must not silence the + // per-case PASS/FAIL lines and human summary. + if (token === "--json-out") args.jsonOut = value; if (token === "--mode") { if (!["combined", "quality", "latency"].includes(value)) throw new Error("--mode must be combined, quality, or latency."); @@ -227,50 +230,6 @@ function normalizedDocumentName(value: string) { .trim(); } -const clinicalDocumentAliases: Record = { - AgitationArousalPharmaMgt: [ - "Agitation and Arousal Pharmacological Management", - "Pharmacological Management of Acute Agitation and Arousal", - "Medication for Agitation and Arousal", - // The corpus has two legitimate agitation IM/PO guidelines. Once the full hybrid stack was - // restored, "Mental Health Pharmacological Management of Agitation and Arousal Guideline (EMHS)" - // ranks alongside/above MHSP.AgitationArousalPharmaMgt for agitation-med queries. Both are - // correct sources, so either satisfies the expectation. (Doc crowding/lexical-weighting for the - // pinned doc is tracked separately as a ranking item, not a retrieval miss.) - "Pharmacological Management of Agitation and Arousal", - ], - AdmissionCommunityPts: ["Admission of Community Patients", "Admission Community Patients"], - ActiveCommunityPtED: [ - "Active Community Patients in the Emergency Department", - "Active Community Patients Emergency Department", - ], - ClozapinePresAdminMonitor: [ - "Clozapine Prescribing Administration Monitoring", - "Clozapine Prescribing Administration and Monitoring", - "Clozapine Prescribing Administering Monitoring", - "Clozapine Prescribing Administering Monitoring and Capillary Sampling", - ], - PtSafetyPlan: ["Patient Safety Plan"], -}; - -const clinicalContentAliases: Record = { - anc: ["anc", "absolute neutrophil count", "neutrophil", "neutrophils"], - // The scale's written name is the hyphenated token "CIWA-Ar"; textContainsClinicalTerm is - // whitespace-delimited, so the bare fixture term can never match it (canary runs #50/#51: - // the top-5 included the CIWA-Ar dosing-table region yet the content gate reported a miss). - ciwa: ["ciwa", "ciwa-ar"], - fbc: ["fbc", "full blood count", "full blood", "wbc", "white blood cell", "white cell"], - im: ["im", "intramuscular", "intramuscularly"], - mg: ["mg", "milligram", "milligrams", "dose", "doses"], - microgram: ["microgram", "micrograms", "mcg", "dose", "doses"], - po: ["po", "oral", "orally"], - prn: ["prn", "as required"], - red: ["red", "red zone", "high risk", "visual alert", "aggression risk"], - route: ["route", "oral", "orally", "intramuscular", "intramuscularly", "im", "po"], - threshold: ["threshold", "below", "drops below", "between", "less than"], - withhold: ["withhold", "withheld", "withholding", "cease", "ceased", "stop", "stopped", "red"], -}; - function contentExpectationAlternatives(expectation: GoldenRetrievalCase["expectedContentTerms"][number]) { const terms = Array.isArray(expectation) ? expectation : [expectation]; return Array.from( @@ -1016,7 +975,7 @@ async function main() { : "", ].filter(Boolean) : []; - if (args.json) { + if (args.json || args.jsonOut) { const payload = { fixture: args.fixture, mode: args.mode, @@ -1030,8 +989,9 @@ async function main() { mkdirSync(dirname(args.jsonOut), { recursive: true }); writeFileSync(args.jsonOut, `${json}\n`); } - console.log(json); - } else { + if (args.json) console.log(json); + } + if (!args.json) { printHumanSummary(summary); if (latencyThresholdFailures.length) { console.log(""); diff --git a/scripts/lib/clinical-aliases.ts b/scripts/lib/clinical-aliases.ts new file mode 100644 index 000000000..4e9019cdf --- /dev/null +++ b/scripts/lib/clinical-aliases.ts @@ -0,0 +1,72 @@ +// Sanctioned alias tables for golden-eval expectations — the single source of truth shared by +// the live eval gates (scripts/eval-retrieval.ts) and the offline ranking-snapshot builder +// (scripts/build-ranking-snapshot.ts via scripts/lib/ranking-snapshot-builder.ts). +// +// Every entry is deliberate drift absorption for ground-truth labels: a fixture keys a case on a +// pinned document name or clinical term, and the corpus legitimately serves the same content under +// another name. Adding an alias is a clinical-governance change — it widens what counts as a hit +// for both the live gates and the snapshot's relevance grades — so each entry needs the same +// evidence trail as a fixture edit (see the per-entry comments). + +/** + * Document-name aliases, keyed by the fixture's `expectedDocumentSubstrings` entry (verbatim). + * Values are alternative title/file-name substrings that identify the same clinical document. + */ +export const clinicalDocumentAliases: Record = { + AgitationArousalPharmaMgt: [ + "Agitation and Arousal Pharmacological Management", + "Pharmacological Management of Acute Agitation and Arousal", + "Medication for Agitation and Arousal", + // The corpus has two legitimate agitation IM/PO guidelines. Once the full hybrid stack was + // restored, "Mental Health Pharmacological Management of Agitation and Arousal Guideline (EMHS)" + // ranks alongside/above MHSP.AgitationArousalPharmaMgt for agitation-med queries. Both are + // correct sources, so either satisfies the expectation. (Doc crowding/lexical-weighting for the + // pinned doc is tracked separately as a ranking item, not a retrieval miss.) + "Pharmacological Management of Agitation and Arousal", + ], + AdmissionCommunityPts: ["Admission of Community Patients", "Admission Community Patients"], + ActiveCommunityPtED: [ + "Active Community Patients in the Emergency Department", + "Active Community Patients Emergency Department", + ], + ClozapinePresAdminMonitor: [ + "Clozapine Prescribing Administration Monitoring", + "Clozapine Prescribing Administration and Monitoring", + "Clozapine Prescribing Administering Monitoring", + "Clozapine Prescribing Administering Monitoring and Capillary Sampling", + ], + PtSafetyPlan: ["Patient Safety Plan"], +}; + +/** + * Content-term aliases, keyed by the lowercase whitespace-collapsed form of a fixture + * `expectedContentTerms` entry. Values are equivalent clinical spellings/expansions. + */ +export const clinicalContentAliases: Record = { + anc: ["anc", "absolute neutrophil count", "neutrophil", "neutrophils"], + // The scale's written name is the hyphenated token "CIWA-Ar"; textContainsClinicalTerm is + // whitespace-delimited, so the bare fixture term can never match it (canary runs #50/#51: + // the top-5 included the CIWA-Ar dosing-table region yet the content gate reported a miss). + ciwa: ["ciwa", "ciwa-ar"], + fbc: ["fbc", "full blood count", "full blood", "wbc", "white blood cell", "white cell"], + im: ["im", "intramuscular", "intramuscularly"], + mg: ["mg", "milligram", "milligrams", "dose", "doses"], + microgram: ["microgram", "micrograms", "mcg", "dose", "doses"], + po: ["po", "oral", "orally"], + prn: ["prn", "as required"], + red: ["red", "red zone", "high risk", "visual alert", "aggression risk"], + route: ["route", "oral", "orally", "intramuscular", "intramuscularly", "im", "po"], + threshold: ["threshold", "below", "drops below", "between", "less than"], + withhold: ["withhold", "withheld", "withholding", "cease", "ceased", "stop", "stopped", "red"], +}; + +/** The fixture label plus every sanctioned document alias, in match-priority order. */ +export function documentLabelAlternatives(label: string): string[] { + return [label, ...(clinicalDocumentAliases[label] ?? [])]; +} + +/** The fixture term plus every sanctioned content alias (alias keys are normalized lowercase). */ +export function contentTermAlternatives(term: string): string[] { + const key = term.toLowerCase().replace(/\s+/g, " ").trim(); + return Array.from(new Set([term, ...(clinicalContentAliases[key] ?? [])])); +} diff --git a/scripts/lib/ranking-snapshot-builder.ts b/scripts/lib/ranking-snapshot-builder.ts index 68b55fd77..5390e4e58 100644 --- a/scripts/lib/ranking-snapshot-builder.ts +++ b/scripts/lib/ranking-snapshot-builder.ts @@ -1,3 +1,4 @@ +import { contentTermAlternatives, documentLabelAlternatives } from "./clinical-aliases"; import type { RankingCandidateFeatures } from "./ranking-tuning"; export type ArtifactScoreExplanation = { @@ -48,6 +49,29 @@ export function labelMatches(text: string, label: string): boolean { .some((part) => normalizedText.includes(` ${part} `)); } +/** + * Alias-aware document matching: a candidate identifies the expected document when its + * title/file text token-matches the fixture label or any sanctioned document alias. Mirrors + * the live eval's documentExpectationAlternatives so snapshot relevance grades agree with the + * gates — without this, alias-satisfied hits (e.g. the EMHS agitation guideline) grade as + * misses and the tuner learns from mislabeled data. + */ +export function documentLabelMatchesWithAliases(documentText: string, label: string): boolean { + return documentLabelAlternatives(label).some((alternative) => labelMatches(documentText, alternative)); +} + +/** + * Alias-aware content matching: artifact content labels are contentExpectationLabel output — + * "a OR b" alternates — and each alternate additionally expands through the sanctioned + * content aliases (e.g. "ciwa" also matches the corpus's hyphenated "CIWA-Ar"). + */ +export function contentLabelMatchesWithAliases(contentText: string, label: string): boolean { + return label + .split(/\s+OR\s+/i) + .filter(Boolean) + .some((part) => contentTermAlternatives(part).some((alternative) => labelMatches(contentText, alternative))); +} + function lexicalContribution(coverage: number): number { const density = Math.min(0.08, Math.max(0, coverage) * 0.08); const direct = coverage >= 0.75 ? 0.08 : coverage >= 0.5 ? 0.035 : coverage <= 0.2 ? -0.08 : 0; diff --git a/scripts/lib/ranking-tuning.ts b/scripts/lib/ranking-tuning.ts index 15922e7b9..633ee1ea4 100644 --- a/scripts/lib/ranking-tuning.ts +++ b/scripts/lib/ranking-tuning.ts @@ -50,6 +50,10 @@ export type RankingSnapshot = { schema: "rag-ranking-candidate-snapshot"; version: typeof RANKING_SNAPSHOT_VERSION; sourceCaseCount: number; + /** ISO timestamp of snapshot generation; drives the 30-day freshness gate once present. */ + generatedAt?: string; + /** GitHub Actions run id of the eval-canary run whose artifact produced this snapshot. */ + sourceRunId?: string; sanitization: { candidateIdentity: "sha256"; excludes: string[]; @@ -120,12 +124,22 @@ export function validateRankingSnapshot(value: unknown): RankingSnapshot { if (!isRecord(value)) throw new Error("Ranking snapshot must be an object"); if (value.schema !== "rag-ranking-candidate-snapshot") throw new Error("Unsupported ranking snapshot schema"); if (value.version !== RANKING_SNAPSHOT_VERSION) throw new Error("Unsupported ranking snapshot version"); - if (!Array.isArray(value.cases) || value.cases.length !== 36) { - throw new Error("Ranking snapshot must contain exactly 36 cases"); + // Floor, not an exact pin: the golden fixture only grows (36 at introduction), and fewer + // cases means a truncated artifact that must not become the tuner's ground truth. + if (!Array.isArray(value.cases) || value.cases.length < 36) { + throw new Error("Ranking snapshot must contain at least 36 cases"); } if (value.sourceCaseCount !== value.cases.length) { throw new Error("Ranking snapshot sourceCaseCount must match cases.length"); } + if (value.generatedAt !== undefined) { + if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { + throw new Error("Ranking snapshot generatedAt must be an ISO date-time string when present"); + } + } + if (value.sourceRunId !== undefined && (typeof value.sourceRunId !== "string" || !value.sourceRunId)) { + throw new Error("Ranking snapshot sourceRunId must be a non-empty string when present"); + } const sanitization = value.sanitization; if (!isRecord(sanitization) || sanitization.candidateIdentity !== "sha256") { throw new Error("Ranking snapshot sanitization.candidateIdentity must be sha256"); diff --git a/tests/ranking-tuning.test.ts b/tests/ranking-tuning.test.ts index 41ab53d10..3880cdf3e 100644 --- a/tests/ranking-tuning.test.ts +++ b/tests/ranking-tuning.test.ts @@ -8,7 +8,12 @@ import { tuneRankingSnapshot, validateRankingSnapshot, } from "../scripts/lib/ranking-tuning"; -import { candidateFeatures, labelMatches } from "../scripts/lib/ranking-snapshot-builder"; +import { + candidateFeatures, + contentLabelMatchesWithAliases, + documentLabelMatchesWithAliases, + labelMatches, +} from "../scripts/lib/ranking-snapshot-builder"; import { defaultRankingConfig, neutralRankingFeatureWeights } from "../src/lib/ranking-config"; const snapshotPath = resolve("scripts/fixtures/rag-ranking-candidate-snapshot.v1.json"); @@ -22,6 +27,25 @@ describe("offline ranking candidate snapshot", () => { expect(labelMatches("Full-blood-count monitoring", "full blood count")).toBe(true); }); + it("grades matches through the sanctioned alias tables the live eval gates use", () => { + // Discriminating document case: the EMHS agitation guideline satisfies the pinned MHSP + // fixture key only via clinicalDocumentAliases — raw labelMatches grades it a miss, which + // fed the tuner mislabeled ground truth before the builder became alias-aware. + const emhsTitle = "mental health pharmacological management of agitation and arousal guideline (emhs).pdf"; + expect(labelMatches(emhsTitle, "AgitationArousalPharmaMgt")).toBe(false); + expect(documentLabelMatchesWithAliases(emhsTitle, "AgitationArousalPharmaMgt")).toBe(true); + expect(documentLabelMatchesWithAliases("clozapine prescribing guideline", "AgitationArousalPharmaMgt")).toBe(false); + + // Discriminating content case: a preview spelling out "absolute neutrophil count" answers + // the "anc" expectation only through clinicalContentAliases. + const ancPreview = "if the absolute neutrophil count drops below 1.5 withhold clozapine"; + expect(labelMatches(ancPreview, "anc")).toBe(false); + expect(contentLabelMatchesWithAliases(ancPreview, "anc")).toBe(true); + // OR-alternate labels (contentExpectationLabel output) expand aliases per part. + expect(contentLabelMatchesWithAliases("give medication as required for agitation", "prn OR stat")).toBe(true); + expect(contentLabelMatchesWithAliases("alcohol withdrawal overview page", "ciwa")).toBe(false); + }); + it("uses exact runtime fusion signals before legacy aggregate reconstruction", () => { const fusionSignals = { hybridRelevance: 0.71, @@ -45,6 +69,27 @@ describe("offline ranking candidate snapshot", () => { ).toEqual(fusionSignals); }); + it("is regenerated within the 30-day freshness window once provenance exists", () => { + // The pre-provenance checked-in snapshot has no generatedAt; the gate activates on the + // first regeneration (build:ranking-snapshot stamps it) and then blocks silent corpus + // drift from an aging snapshot. + if (!snapshot.generatedAt) return; + const ageMs = Date.now() - Date.parse(snapshot.generatedAt); + const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + const regenerate = + "Regenerate it: download the latest eval-canary run's eval-canary-output artifact, then " + + "`npm run build:ranking-snapshot -- --input golden-retrieval.json --output " + + "scripts/fixtures/rag-ranking-candidate-snapshot.v1.json --source-run-id `."; + expect( + ageMs, + `Ranking snapshot generatedAt (${snapshot.generatedAt}) is ${Math.round(ageMs / 86_400_000)} days old. ${regenerate}`, + ).toBeLessThanOrEqual(thirtyDaysMs); + expect( + ageMs, + `Ranking snapshot generatedAt (${snapshot.generatedAt}) is in the future. ${regenerate}`, + ).toBeGreaterThanOrEqual(-86_400_000); + }); + it("is versioned, complete, and excludes raw candidate/source data", () => { expect(snapshot.version).toBe(RANKING_SNAPSHOT_VERSION); expect(snapshot.cases).toHaveLength(36); @@ -93,6 +138,24 @@ describe("offline ranking candidate snapshot", () => { invalidSourceCount.sourceCaseCount = 35; expect(() => validateRankingSnapshot(invalidSourceCount)).toThrow(/sourceCaseCount/); + const truncated = structuredClone(snapshot); + truncated.cases = truncated.cases.slice(0, 35); + truncated.sourceCaseCount = 35; + expect(() => validateRankingSnapshot(truncated)).toThrow(/at least 36/); + + const invalidGeneratedAt = structuredClone(snapshot); + invalidGeneratedAt.generatedAt = "not-a-date"; + expect(() => validateRankingSnapshot(invalidGeneratedAt)).toThrow(/generatedAt/); + + const invalidSourceRunId = structuredClone(snapshot); + invalidSourceRunId.sourceRunId = ""; + expect(() => validateRankingSnapshot(invalidSourceRunId)).toThrow(/sourceRunId/); + + const withProvenance = structuredClone(snapshot); + withProvenance.generatedAt = "2026-07-20T00:00:00.000Z"; + withProvenance.sourceRunId = "16412345678"; + expect(validateRankingSnapshot(withProvenance).generatedAt).toBe("2026-07-20T00:00:00.000Z"); + const invalidSanitization = structuredClone(snapshot); invalidSanitization.sanitization.candidateIdentity = "raw" as never; expect(() => validateRankingSnapshot(invalidSanitization)).toThrow(/sanitization/); @@ -129,10 +192,11 @@ describe("offline ranking tuner", () => { it("keeps the golden-regression hard negatives below the first relevant candidate at production weights", () => { // The four cases that failed the 2026-07-19 live golden retrieval eval on the raw #901 // ordering (remediated same-day by #913-#926). This gate keys on relevanceGrade-derived - // metrics (missing positives, hard-negative ordering), not on documentMatch flags, so the - // snapshot builder's alias-free labelMatches cannot weaken it. It exercises the snapshot - // proxy scorer at production weights — a complementary floor, not a test of the - // selectRetrievalEvidence comparator (tests/rag-fast-path-ordering.test.ts covers that). + // metrics (missing positives, hard-negative ordering), not on documentMatch flags, so + // grading changes in the (now alias-aware) snapshot builder only reach it through a + // deliberate snapshot regeneration. It exercises the snapshot proxy scorer at production + // weights — a complementary floor, not a test of the selectRetrievalEvidence comparator + // (tests/rag-fast-path-ordering.test.ts covers that). const goldenRegressionCaseIds = [ "lithium-therapy-monitoring", "clozapine-anc-threshold", From 2e82b97d2efa4214f603fe393c71af0f33777e36 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 16:54:20 +0000 Subject: [PATCH 2/2] docs(ledger): record eval measurement-floor review row (A-PR-1) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index f2b5e4fa9..96a7d9f1d 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -650,3 +650,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: coverage tie-break follow-up) | 57ec880b306a8e2b31c5f20dacc47256fc93b4e2 | Post-merge live-eval finding on #982 + corrective fix: saturated-tie key rankScore → query-term coverage | Post-#982 golden dispatch (eval-canary run #50, 29735004222, main b9057f0 + deps) came back 35/36: the three verifiable July-19 failures (lithium-therapy-monitoring, clozapine-anc-threshold, patient-safety-plan-include) all PASS live, but alcohol-ciwa-threshold flipped pass→FAIL vs the same-morning pre-#982 run #49 (29731533081, 36/36) — failing top-3 ordered by descending rankScore (1.85/1.75/1.53, all finalScore-saturated, releaseRankScore 1.09/1.086/1.07), i.e. #982's tie-break let generic clinicalSignalBoost stacking outvote the ciwa/score/threshold-bearing chunk; #982 is the only retrieval-path delta in the window. Fix: contentRankScore → contentCoverageScore sourced from lexicalCoverageScore (query-term coverage, immune to boost stacking; ties still fall to chunk id); saturated-tie contract test re-pinned so coverage beats a HIGHER rankScore (discriminating — old key fails it); fast-path CIWA guard gains the run-#50 screening-chunk shape + content-term assertion. Live validation: tonight's 18:00 UTC scheduled canary (dispatch cap 2/2 spent ≈$2-4). Separately: ci.yml dispatch 4012 survived 30+ min of main churn under #979's per-run concurrency group (fix working); duplicate dispatch 4017 cancelled. | Targeted vitest 38/38; npm run test 3012 passed / 1 known container-only pdf-budget artifact; verify:cheap green to the same artifact; build + client-bundle scan + check:rag:fixtures PASS; check:production-readiness expected missing-secret FAILs only (no secrets in container) | | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: WebKit matrix diagnosis) | caa298972846ef52d472bfbff98c90fc26481ed8 | First completed release-browser-matrix in weeks (run 4012, main b9057f0, 38 min uncancelled under #979) — full triage of 28 failures + fix for the dominant cluster | Run 4012: 716 passed / 28 failed (25 webkit, 3 firefox) / 4 skipped. Dominant cluster (root-caused, FIXED here): all 8 ui-universal-search webkit failures share one signature — typeahead content never enters the DOM — because commandDropdownCanDisplay (added 42a3e3c 2026-07-17, AFTER the last completed matrix; never ran on WebKit until 4012) requires fine-pointer OR zero-touch; headless browsers fail the (hover:hover)+(pointer:fine) query (proven by the zero-touch escape existing for CI at all), Chromium/Firefox pass via maxTouchPoints===0, and Playwright's Linux WebKit build advertises phantom touch points — flunking both branches and disabling useUniversalSearch entirely. Fix: beforeEach addInitScript in ui-universal-search/ui-smoke/ui-tools specs stubbing Navigator.prototype.maxTouchPoints to the runner's true 0 (inert on Chromium/Firefox; product gate + tests/search-command-surface.test.ts pins untouched). Local chromium runs of ui-universal-search fail 16/20 IDENTICALLY on unmodified main (container artifact — hosted CI chromium green in 4012 is authoritative; verified by stash/run/pop baseline). Expected delta next matrix: ≥8 webkit failures clear; candidates ui-tools:1244 + several ui-smoke answer-flow cases (same surface). NOT yet root-caused (triaged remainder, hosted-matrix-only reproduction): webkit ui-stress overflow 330/409 (360px overflow at mobile), webkit ui-accessibility 195/252 (focus dismissal; forced-colors labels), webkit ui-formulation 132, webkit ui-smoke copy-table/retry/recovery/recent-searches/source-only/differential-context/viewer-hydration/document-questions, webkit ui-tools 2087 (goto interrupted by ?q= navigation), webkit ui-universal-search 183 strict-mode duplicate options (fallback-surface rendering; likely clears with the gate fix), firefox ui-smoke 946/2206/2932 (@critical document search fails on BOTH firefox and webkit — cross-browser, highest-priority remainder). | prettier PASS; eslint PASS; typecheck PASS; chromium stash-baseline no-delta (16 fail pre AND post — container artifact); webkit validation lands via tonight's 18:00 UTC scheduled matrix | | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: matrix-remainder fixes) | ec070ccb20b9a067216e64e8399d9cd795027024 | Run-4012 remainder: 7 root-caused fixes (SW navigation hijack, WebKit focus/forced-colors/stale-style, Firefox tab order, CIWA alias, helper extraction) | Agent-diagnosed with probe evidence: (1) production-build PWA worker registers in EVERY matrix test (pwa-lifecycle.tsx:287 NODE_ENV gate), clients.claim()s the page and serves all navigations — Chromium probe proved SW-served reloads bypass page.route entirely (routeSawNav=[]); Playwright-Firefox wedges on its only two reloads (ui-smoke 2206/2932, both deterministic) → playwright.config.ts serviceWorkers:'block' + ui-pwa 'allow' opt-in (offline/CacheStorage journey verified passing locally under the opt-in). (2) Mode-menu dismissal relied solely on wrapper focusout; WebKit Tab navigation can move focus nowhere (links excluded) or wrap into the menu → keydown Tab-close in handleModeTriggerKeyDown (agent verified no test depends on old forward-Tab behavior). (3) WebKit has no forced-colors implementation → capability skip on the token-remap test. (4) WebKit stale :disabled computed style feeds axe a phantom 1.93 contrast for the re-enabled Previous button (blend arithmetic exact: 0.4×#475467+0.6×#fff=#b5bbc2) → toBeEnabled+opacity-1 pin before the scan, failure-at-pin = direct proof. (5) Firefox includes scrollable containers in tab order (sheet body scrollHeight 1125 vs 707 measured) → conditional step-over. (6) canary #50/#51 CIWA failure root cause: whitespace-delimited textContainsClinicalTerm can never match hyphenated 'CIWA-Ar' though the dosing-table region ranks top-5 → clinicalContentAliases ciwa:[ciwa, ciwa-ar] (plan-authorized fixture-alias route; content substance unchanged). (7) tests/helpers/zero-touch.ts shared helper replaces six #995 inline stubs (CodeRabbit follow-up). NOT claimed: ui-stress 409 overflow, ui-tools 2087 navigation race, webkit answer-flow subset — next matrix run (post-merge dispatch) measures these vs the 28-failure baseline; ui-smoke 2932 webkit-side is a distinct mock-data class deliberately deferred pending that run. | Targeted vitest 25/25 (eval-retrieval + search-command-surface); npm run test 3012 passed / 1 known container pdf-budget artifact; verify:cheap green to same artifact; build + client-bundle scan PASS; prettier/eslint/typecheck clean; chromium ui-pwa under new SW config: offline/CacheStorage privacy journey PASS, installability = known container in-incognito artifact; UI verification not run locally beyond that: chromium ui suites carry documented container artifacts — hosted ui-critical/ui-advisory + next matrix authoritative | +| 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: eval measurement floor, ADDENDUM 4 A-PR-1) | 81ab9696da4b330ca0b2e5519891a9942f90421b | Measurement floor for evidence-gated ranking tuning: canary artifact emission, alias-aware snapshot builder, snapshot provenance/freshness — no ranking behavior change | Closes the three gaps blocking safe tuning (Phase B): (1) eval-canary's golden step now writes the per-case JSON artifact (--json-out decoupled from --json so the tee'd log keeps the human-readable lines the failure-issue analyzer parses) and uploads .local/eval-canary/ via pinned upload-artifact (30-day retention, include-hidden-files for the dot-dir, contents = same class as the already-public step logs: titles/telemetry/220-char previews of the all-public corpus) — snapshot regeneration stops costing a paid dispatch; (2) clinicalDocumentAliases/clinicalContentAliases moved verbatim to shared scripts/lib/clinical-aliases.ts and the snapshot builder grades documentMatch/contentMatch through them (discriminating tests: EMHS agitation title and spelled-out "absolute neutrophil count" grade as hits only via aliases — raw labelMatches pinned false), ending tuner ground truth disagreeing with the live gates; (3) snapshots carry generatedAt + optional sourceRunId, validator accepts them, exactly-36 relaxed to at-least-36 (floor still rejects truncated artifacts; sourceCaseCount + per-case candidate minimums unchanged), and a 30-day freshness test (activates on first regeneration) blocks silent corpus drift. Builder smoke-verified end-to-end on a synthetic 36-case artifact (alias grading + provenance stamped + validator green). Static hyphen audit of all 36 cases' terms: no currently-blocked term (canary #52 = 36/36); residual risk classes documented for the A-PR-2 artifact-grounded pass — punctuation-joined tokens (IM/PO, schizo-affective, post-natal) and inert stem entries (obsess/compuls/hyperactiv/impuls can never match whole-token) that currently ride on whole-word OR-alternates. | Targeted vitest 52/52 (ranking-tuning + eval-retrieval + eval-quality); npm run test 3019 passed / 1 known container-only pdf-budget artifact; lint + typecheck clean; check:github-actions + check:ci-scope PASS; prettier clean; check:production-readiness expected missing-secret FAILs only (demo-mode container); no provider calls — live validation = tonight's scheduled canary emits the first artifact at $0 |