From fd5dcd1582e1c7aa8bf92ff097575e0c68aa930d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 09:56:43 +0000 Subject: [PATCH 1/3] fix(rag): break saturated selection ties by content rank, add fast-path ordering guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-07-19 golden retrieval eval failed 4/36 on the raw #901 ordering. The same-day remediation (#913-#926) restored live results (fresh eval on remediated main passed 36/36 today), but the failure class had zero offline coverage and one reproducible defect survived: when the embedding-free text fast path imputes byte-identical primaries, selection's clamped sort keys all tie exactly and the chunk-id fallback decides — an arbitrary order that the second stage's position-based adjustment then launders into releaseRankScore and the released order, burying the answer-bearing document. - retrieval-selection: carry contentRankScore (the content-aware clinical rank) on candidates and use it ONLY as the tie-break between the clamped rerank confidence and the chunk-id fallback. It is never added to a score, so the measured clamped-score contract is unchanged. - tests/rag-fast-path-ordering.test.ts: four end-to-end guards replaying the failed eval cases' shapes through the production ordering pipeline. - tests/ranking-tuning.test.ts: gate the four golden-mapped hard negatives at production weights plus a full-snapshot high-risk assertion. - tests/retrieval-selection.test.ts: amend #901's saturated-tie pin (chunk-id order was never live-eval-validated) to the content-rank-then-id contract, with an id-fallback case for identical content ranks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- src/lib/retrieval-selection.ts | 15 ++ src/lib/types.ts | 1 + tests/rag-fast-path-ordering.test.ts | 309 +++++++++++++++++++++++++++ tests/ranking-tuning.test.ts | 26 +++ tests/retrieval-selection.test.ts | 23 +- 5 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 tests/rag-fast-path-ordering.test.ts diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 35779ca27..74ca25239 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -490,6 +490,15 @@ export function buildRetrievalCandidates( // rank already used rankScore; reusing the unbounded value here would compound boosts across // passes and reintroduce the measured recall regression guarded below. rerankScore: result.score_explanation?.finalScore ?? result.hybrid_score, + // Carried ONLY as the last tie-break before chunk id: on the embedding-free fast path, + // imputed primaries make clamped score/lexical/rerank ties routine, and a chunk-id tie-break + // is arbitrary — the second stage's position-based adjustment then launders that arbitrary + // winner into the released order. Never added to score. + contentRankScore: + result.score_explanation?.rankScore ?? + result.score_explanation?.preClampFinalScore ?? + result.score_explanation?.finalScore ?? + result.hybrid_score, matchedSignals: [], sourceHref: documentCitationHref(citationFromResult(result)), }; @@ -602,6 +611,12 @@ export function selectRetrievalEvidence(args: { if ((right.lexicalScore ?? 0) !== (left.lexicalScore ?? 0)) return (right.lexicalScore ?? 0) - (left.lexicalScore ?? 0); if ((right.rerankScore ?? 0) !== (left.rerankScore ?? 0)) return (right.rerankScore ?? 0) - (left.rerankScore ?? 0); + // Exact tie on every clamped key (routine on the embedding-free fast path, where imputed + // primaries are byte-identical and confidences saturate at 1.0): prefer the content-aware + // clinical rank over an arbitrary chunk-id ordering. This never raises any score — it only + // orders otherwise-indistinguishable candidates, so the measured clamped-score contract holds. + if ((right.contentRankScore ?? 0) !== (left.contentRankScore ?? 0)) + return (right.contentRankScore ?? 0) - (left.contentRankScore ?? 0); return left.chunkId.localeCompare(right.chunkId); }); const selectedCandidates: RetrievalCandidate[] = []; diff --git a/src/lib/types.ts b/src/lib/types.ts index d2441c88b..0299b278c 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -179,6 +179,7 @@ export type RetrievalCandidate = { lexicalScore?: number; semanticScore?: number; rerankScore?: number; + contentRankScore?: number; matchedSignals: string[]; sourceHref?: string; }; diff --git a/tests/rag-fast-path-ordering.test.ts b/tests/rag-fast-path-ordering.test.ts new file mode 100644 index 000000000..4c04783e7 --- /dev/null +++ b/tests/rag-fast-path-ordering.test.ts @@ -0,0 +1,309 @@ +import { describe, expect, it } from "vitest"; +import { applySecondStageRerankIfNeeded } from "../src/lib/rag"; +import { classifyRagQuery, rankClinicalResults } from "../src/lib/clinical-search"; +import { selectRetrievalEvidence } from "../src/lib/retrieval-selection"; +import { resultsHaveReleaseRankScore, stabilizeReleasedSearchOrder } from "../src/lib/released-search-order"; +import type { SearchTelemetry } from "../src/lib/rag-contracts"; +import type { SearchResult } from "../src/lib/types"; + +// Regression guards for the 2026-07-19 live golden-retrieval failures (the raw #901 +// ordering, remediated same-day by #913-#926). On the embedding-free text fast path, +// table/text candidates carry IMPUTED primaries derived only from Postgres text_rank +// (rag-candidate-sources.ts: similarity = 0.62 + min(text_rank, 1) * 0.3), so unrelated +// documents matching the same query terms collapse to byte-identical vector/text/hybrid +// scores and only content-aware ranking can separate them. Each test replays one failed +// eval case's shape through the production ordering pipeline. + +type QueryClass = ReturnType["queryClass"]; + +function imputedResult( + overrides: Partial & Pick, +): SearchResult { + return { + page_number: 5, + chunk_index: 0, + section_heading: null, + image_ids: [], + images: [], + similarity: 0.755, + text_rank: 0.45, + hybrid_score: 0.795, + ...overrides, + }; +} + +// Mirrors the production text-fast-path ordering pipeline: selectRankedRetrievalResults +// (rag.ts — rankClinicalResults + selectRetrievalEvidence) → applySecondStageRerankIfNeeded +// → the recordSearchScoreTelemetry release ordering (stabilizeReleasedSearchOrder keyed on +// resultsHaveReleaseRankScore). Supabase-only hydration and the default-off semantic rerank +// are the only production stages skipped; neither reorders deterministic results. +function runTextFastPathOrdering(args: { + query: string; + queryClass: QueryClass; + candidates: SearchResult[]; + topK: number; + maxResultsPerDocument?: number; +}): SearchResult[] { + const selection = selectRetrievalEvidence({ + query: args.query, + queryClass: args.queryClass, + results: rankClinicalResults(args.query, args.candidates), + topK: args.topK, + maxResultsPerDocument: args.maxResultsPerDocument ?? 4, + }); + const reranked = applySecondStageRerankIfNeeded({ + queryClass: args.queryClass, + results: selection.results, + telemetry: {} as SearchTelemetry, + topK: args.topK, + }); + stabilizeReleasedSearchOrder(reranked, resultsHaveReleaseRankScore(reranked)); + return reranked; +} + +function normalizedDocumentName(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function topFiveEvidenceText(results: SearchResult[]): string { + return results + .slice(0, 5) + .map((result) => + [ + result.content, + ...(result.table_facts ?? []).flatMap((fact) => [fact.action, fact.threshold_value, fact.row_label]), + ] + .filter(Boolean) + .join(" "), + ) + .join(" "); +} + +describe("text-fast-path ordering under imputed identical primaries", () => { + it("ranks the medication-subject document above generic monitoring documents (lithium-therapy-monitoring)", () => { + const query = "What monitoring is required for lithium therapy?"; + const { queryClass } = classifyRagQuery(query); + expect(queryClass).toBe("medication_dose_risk"); + + const wrongSubjectDocs = [ + imputedResult({ + id: "haemophilia-p5", + document_id: "haemophilia-doc", + title: "Acquired Haemophilia A Management", + file_name: "AcquiredHaemophiliaAManagement (FSH).pdf", + section_heading: "Immunosuppressive therapy treatment options", + content: + "Monitor for adverse events during immunosuppressive therapy. Baseline blood pressure and glucose monitoring is required for corticosteroid therapy.", + }), + imputedResult({ + id: "methotrexate-p5", + document_id: "methotrexate-doc", + title: "Methotrexate for IBD Guideline", + file_name: "MethotrexateForIBDGuideline (NMHS).pdf", + section_heading: "Monitoring requirements", + content: + "Full blood count and liver function monitoring is required during methotrexate therapy. Repeat baseline blood tests after dose changes.", + }), + imputedResult({ + id: "cardiac-p5", + document_id: "cardiac-doc", + title: "Continuous Cardiac Monitoring Requirements", + file_name: "ContinuousCardiacMonitoringRequirements (SMHS).pdf", + section_heading: "Monitoring during therapy", + content: + "Continuous cardiac monitoring is required during therapy titration. Record baseline observations and monitor levels every hour.", + }), + ]; + const lithiumDoc = imputedResult({ + id: "lithium-p9", + document_id: "lithium-doc", + title: "Mood Stabiliser Guideline", + file_name: "MoodStabiliserGuideline (SMHS).pdf", + page_number: 9, + content: + "Serum lithium level should be checked 12 hours after the last dose during maintenance therapy, with renal function and thyroid function tests every six months.", + }); + + const candidates = [...wrongSubjectDocs, lithiumDoc]; + const released = runTextFastPathOrdering({ query, queryClass, candidates, topK: 12 }); + + expect(released[0]?.document_id).toBe("lithium-doc"); + expect(released.slice(0, 5).some((result) => /\blithium\b/i.test(result.content))).toBe(true); + + // The selection layer must arm the clinical_subject requirement for monitoring queries + // (#919) and tag subject matches, because the rag.ts second-stage cap keys on these + // exact reasons to stop wrong-medication chunks outranking the requested subject. + const haemophilia = released.find((result) => result.document_id === "haemophilia-doc"); + const lithium = released.find((result) => result.document_id === "lithium-doc"); + expect(haemophilia?.match_explanation?.reasons).toContain("retrieval_required_signal:clinical_subject"); + expect(haemophilia?.match_explanation?.reasons).not.toContain("retrieval_signal:clinical_subject"); + expect(lithium?.match_explanation?.reasons).toContain("retrieval_signal:clinical_subject"); + + // Byte-identical primaries must not make the outcome depend on candidate arrival order. + const releasedReversed = runTextFastPathOrdering({ + query, + queryClass, + candidates: [...candidates].reverse(), + topK: 12, + }); + expect(releasedReversed[0]?.document_id).toBe("lithium-doc"); + }); + + it("keeps the threshold table chunk in the released top 5 of its own document (clozapine-anc-threshold)", () => { + const query = "What ANC or FBC threshold should withhold clozapine?"; + const { queryClass } = classifyRagQuery(query); + expect(queryClass).toBe("table_threshold"); + + const proseSiblings = [ + "Clozapine requires regular ANC and FBC checks during the first 18 weeks of treatment.", + "Community pharmacy dispensing rules for clozapine limit supply to short intervals.", + "Clozapine registration and consent must be recorded before commencing treatment.", + "Patients taking clozapine should report infection symptoms so an FBC can be arranged.", + ].map((content, index) => + imputedResult({ + id: `clozapine-prose-${index + 1}`, + document_id: "clozapine-doc", + title: "Clozapine Prescribing Administration Monitoring", + file_name: "MHSP.ClozapinePrescribingAdministrationMonitoring.pdf", + page_number: index + 4, + chunk_index: index + 1, + content, + }), + ); + const tableChunk = imputedResult({ + id: "clozapine-anc-table", + document_id: "clozapine-doc", + title: "Clozapine Prescribing Administration Monitoring", + file_name: "MHSP.ClozapinePrescribingAdministrationMonitoring.pdf", + page_number: 3, + chunk_index: 5, + section_heading: "ANC monitoring thresholds", + content: "ANC below 1.5 requires immediate review as set out in the monitoring table.", + table_facts: [ + { + id: "fact-anc-1", + document_id: "clozapine-doc", + source_chunk_id: "clozapine-anc-table", + source_image_id: null, + page_number: 3, + table_title: "Clozapine ANC monitoring thresholds", + row_label: "ANC < 1.5", + clinical_parameter: "ANC", + threshold_value: "< 1.5 x 10^9/L", + action: "Withhold clozapine and repeat FBC", + }, + ], + }); + const crossDocDistractor = imputedResult({ + id: "fbc-policy-p2", + document_id: "fbc-policy-doc", + title: "Full Blood Count Collection Policy", + file_name: "FullBloodCountCollectionPolicy (NMHS).pdf", + page_number: 2, + content: "FBC collection, labelling and processing procedure for inpatient units.", + }); + + const released = runTextFastPathOrdering({ + query, + queryClass, + candidates: [...proseSiblings, tableChunk, crossDocDistractor], + topK: 12, + // Production caps non-comparison retrieval at 4 chunks per document: if the table + // chunk loses to its prose siblings it is CUT entirely — the failure being guarded. + maxResultsPerDocument: 4, + }); + + const topFive = released.slice(0, 5); + const tableIndex = topFive.findIndex((result) => result.id === "clozapine-anc-table"); + expect(tableIndex).toBeGreaterThanOrEqual(0); + for (const prose of proseSiblings) { + const proseIndex = topFive.findIndex((result) => result.id === prose.id); + if (proseIndex >= 0) { + expect(tableIndex).toBeLessThan(proseIndex); + } + } + expect(topFive.some((result) => (result.table_facts?.length ?? 0) > 0)).toBe(true); + // The eval's content gate also reads table-fact action text (resultContentEvidenceText). + expect(topFiveEvidenceText(released)).toMatch(/withhold|cease|stop/i); + }); + + it("prefers the scale document naming the threshold over the broader AOD overview (alcohol-ciwa-threshold)", () => { + const query = "What CIWA-Ar score threshold requires drug treatment in alcohol withdrawal?"; + const { queryClass } = classifyRagQuery(query); + expect(queryClass).toBe("table_threshold"); + + const ciwaDoc = imputedResult({ + id: "ciwa-p2", + document_id: "ciwa-doc", + title: "Alcohol Withdrawal Scale (CIWA-Ar)", + file_name: "Alcohol Withdrawal Scale (CIWA-Ar) (NMHS).pdf", + page_number: 2, + section_heading: "CIWA-Ar score thresholds", + content: "A CIWA-Ar score of 10 or more requires drug treatment for alcohol withdrawal.", + }); + const aodOverview = imputedResult({ + id: "aod-p6", + document_id: "aod-doc", + title: "Alcohol and Other Drugs - Addiction, Toxicity and Withdrawal", + file_name: "Alcohol and Other Drugs - Addiction, Toxicity and Withdrawal (FSH).pdf", + page_number: 6, + content: + "Assessment of intoxication, toxicity and withdrawal for alcohol and other drugs, with supportive care and monitoring principles.", + }); + + // Offline this pins the pairwise decision between the two live contenders; the live + // corpus depth (the expected document sat at rank ~8 on 2026-07-19) is covered by the + // live golden eval, not this fixture. + const released = runTextFastPathOrdering({ + query, + queryClass, + candidates: [aodOverview, ciwaDoc], + topK: 12, + }); + + expect(released[0]?.document_id).toBe("ciwa-doc"); + const docGateSatisfied = released + .slice(0, 5) + .some((result) => normalizedDocumentName(`${result.title} ${result.file_name}`).includes("alcohol withdrawal")); + expect(docGateSatisfied).toBe(true); + }); + + it("ranks the pinned safety-plan document above the policy duplicate for document lookups (patient-safety-plan-include)", () => { + const query = "What should a patient safety plan include?"; + const { queryClass } = classifyRagQuery(query); + expect(queryClass).toBe("document_lookup"); + + const pinnedPlan = imputedResult({ + id: "ptsafetyplan-p1", + document_id: "ptsafetyplan-doc", + title: "Patient Safety Plan", + file_name: "PtSafetyPlan (MHSP).pdf", + page_number: 1, + content: + "A patient safety plan should include warning signs, coping strategies, social contacts, and emergency contacts.", + }); + const policyDuplicate = imputedResult({ + id: "rkpg-policy-p3", + document_id: "rkpg-policy-doc", + title: "Safety Planning Policy and Procedure", + file_name: "Safety Planning Policy and Procedure (RKPG).pdf", + page_number: 3, + content: "Safety planning policy describing what a safety plan should include for patients at risk.", + }); + + const released = runTextFastPathOrdering({ + query, + queryClass, + candidates: [policyDuplicate, pinnedPlan], + topK: 8, + }); + + expect(released[0]?.document_id).toBe("ptsafetyplan-doc"); + // Both are legitimate sources — the policy document must stay retrievable, not be dropped. + expect(released.map((result) => result.document_id)).toContain("rkpg-policy-doc"); + }); +}); diff --git a/tests/ranking-tuning.test.ts b/tests/ranking-tuning.test.ts index 30c2fd85d..9e0cac1cc 100644 --- a/tests/ranking-tuning.test.ts +++ b/tests/ranking-tuning.test.ts @@ -126,6 +126,32 @@ describe("offline ranking tuner", () => { expect(metrics.highRiskHardNegativeFailures).toBe(0); }); + it("keeps the golden-regression hard negatives below the first relevant candidate at production weights", () => { + // The four cases that failed the 2026-07-19 live golden retrieval eval on the raw #901 + // ordering (remediated same-day by #913-#926). Snapshot-builder caveat: labelMatches does + // not apply the eval's clinicalDocumentAliases, so alias-keyed documentMatch flags are + // false snapshot-wide — the content-graded relevant candidates carry this gate instead. + const goldenRegressionCaseIds = [ + "lithium-therapy-monitoring", + "clozapine-anc-threshold", + "alcohol-ciwa-threshold", + "patient-safety-plan-include", + ]; + for (const caseId of goldenRegressionCaseIds) { + const testCase = snapshot.cases.find((item) => item.id === caseId); + expect(testCase, caseId).toBeDefined(); + const metrics = evaluateRankingCases([testCase!], defaultRankingConfig.featureFusion[testCase!.queryClass]); + expect(metrics.missingPositiveCases, caseId).toBe(0); + expect(metrics.highRiskHardNegativeFailures, caseId).toBe(0); + expect(metrics.hardNegativeAccuracy, caseId).toBe(1); + } + }); + + it("keeps the full snapshot free of high-risk hard-negative failures at neutral weights", () => { + const metrics = evaluateRankingCases(snapshot.cases, neutralRankingFeatureWeights); + expect(metrics.highRiskHardNegativeFailures).toBe(0); + }); + it("is deterministic and only selects constrained improvements", () => { const first = tuneRankingSnapshot(snapshot); const second = tuneRankingSnapshot(snapshot); diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index 3889ddf1d..2e4b6230a 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -682,7 +682,13 @@ describe("saturated-score tie-breaking", () => { }; } - it("keeps the eval-validated stable order for saturated selection scores", () => { + it("breaks saturated-score ties by content-aware clinical rank, then chunk id", () => { + // Amended from #901's chunk-id-only pin (which was never live-eval-validated — the only + // golden eval on the #901 state failed 4/36): among candidates whose clamped score, lexical + // coverage, and clamped rerank confidence all tie exactly, the content-aware clinical rank + // now decides before the chunk-id fallback. On the embedding-free fast path this is what + // stops an arbitrary id ordering from burying the answer-bearing document + // (alcohol-ciwa-threshold class; see tests/rag-fast-path-ordering.test.ts). const higherPreClamp = source({ id: "chunk-b", hybrid_score: 1, @@ -704,8 +710,21 @@ describe("saturated-score tie-breaking", () => { maxResultsPerDocument: 2, }); - expect(selection.results.map((item) => item.id)).toEqual(["chunk-a", "chunk-b"]); + expect(selection.results.map((item) => item.id)).toEqual(["chunk-b", "chunk-a"]); // The prior recall fix remains: selection never lowers the raw hybrid score. expect(selection.results.map((item) => item.hybrid_score)).toEqual([1, 1]); + + // Identical content rank still falls back to the stable chunk-id order. + const tiedSelection = selectRetrievalEvidence({ + query: "clinical guidance", + queryClass: "broad_summary", + results: [ + source({ id: "chunk-d", hybrid_score: 1, similarity: 0.9, score_explanation: saturatedExplanation(1.4) }), + source({ id: "chunk-c", hybrid_score: 1, similarity: 0.9, score_explanation: saturatedExplanation(1.4) }), + ], + topK: 2, + maxResultsPerDocument: 2, + }); + expect(tiedSelection.results.map((item) => item.id)).toEqual(["chunk-c", "chunk-d"]); }); }); From 84a5381b0326f4b3624479138ebae54e5e56eb05 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 09:57:01 +0000 Subject: [PATCH 2/3] docs(ledger): record #901 regression closeout review row Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 95ebaac07..9c638bf1b 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -645,3 +645,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-20 | origin/main PWA-surface review, window `ef042ca..e128384` (44+ commits; explicit user request) + phone install-sheet redesign (this branch, content commit d8d8c4a) | e1283846647b20cd49ca6ae6920a7c76f2b945d7 | PWA-version review of new progress + install-notification design elevation | Sweep verdict (agent-verified): only #905/#897 (this program's own work) touched PWA surfaces in the window — sw.js CACHE_VERSION `2026-07-18-v1` and its offline.html binding, manifest (7 icons incl. monochrome; screenshots deliberately absent), icons route, kill-switch, and next.config headers all unchanged, so the privacy contract holds by construction. Geometric risk from five composer/dock-space reworks (#933 mobile-composer-reserve, #932, #930, #922, #899) probed live at 390×844: the non-install notice stack keeps correct clearance at rest (offline card bottom 744/844 with the 5.5rem gutter) and the stack's offset is now a custom property (`--pwa-notice-bottom-gap`) for one-line re-anchoring if composer geometry moves again. Redesign: install prompt + iOS hint present as a native bottom sheet on phones (full-bleed, flush bottom, grip bar matching #935's sheet language, real app icon identity via /icons/icon-192, Free/no-store meta row, two accent step chips for Share→Add to Home Screen as aria-hidden reinforcement of the unchanged accessible sentence); ≥640px keeps the #905 card/toast placements; region names, button names, and test-locked sentences unchanged. | Focused vitest 56/56 across the four PWA suites; `verify:cheap` 2971/2975 (sole fail = known container-only pdf-extraction-budget artifact); `test:e2e:pwa` privacy/offline green (installability fail = known container `in-incognito` artifact); `verify:ui` 243 passed/2 failed (the two long-baselined container artifacts); production build + client-bundle secret scan + bundle budget within tolerance (1303.4 vs 1278.6 KiB baseline; sheet adds ~0 JS); visual evidence at 390 light/dark/iOS-hint + 768 + 1440 with bounding-box math (full-bleed flush-bottom on phone; card/toast intact above). No provider-backed checks run. | | 2026-07-20 | Credentialed release-gate checkpoint closeout (workflow_dispatch on main `7ec25d9`; user-authorized ≤$10, single dispatch each) | 7ec25d9675dea13635fa4a895af88c93da694a42 | Credentialed half of the release gate: CI dispatch + live eval canary | CI dispatch: 9/10 jobs green (unit coverage, build, Chromium production journeys, migration replay on local Supabase emulator, production-readiness CI-safe, policy self-tests, static/safety/scope). `release-browser-matrix` (WebKit/Firefox) CANCELLED twice by main-churn: ci.yml `concurrency: CI-${ref}, cancel-in-progress: true` kills in-flight dispatch runs on every main push and this repo merges every few minutes — livelock confirmed at the 2-attempt cap; WebKit/iOS verification remains outstanding with three human options (quiet-window dispatch, the weekly scheduled run, or a one-line dedicated concurrency group for the matrix job — operational-risk change, not applied). Eval canary: golden retrieval eval FAILED 4/36 (document_recall@5 0.944, ndcg@10 0.923, force_embedding_failure_count 0, no 429s — vector layer healthy, NOT the documented vector-ptsd transient class); the July 17 dispatch PASSED this step pre-#901, so the regression window implicates #901's deterministic semantic reranking (lithium-therapy-monitoring shows three unrelated documents with byte-identical rerank scores burying the lithium guideline; two other failures add fixture-vs-corpus identity components; answer-quality subset — the July 17 failure — never ran). NO canary re-run per plan (deterministic, not transient); retrieval is clinical-path and deferred to a human decision. Provider spend ≈ $1–2 (one canary run's embeddings); matrix/CI runs $0. | actions_run_trigger dispatches + rerun_failed_jobs (attempt 2); job-level conclusions and log excerpts from runs 29675875530 (both attempts), 29675878737 (July 19 canary), and 29567502452 (July 17 baseline). No local provider calls; secrets never left GitHub Actions. | | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: ci.yml concurrency) | 79aaacf04e64f753c480dd957a81ac9be6acbb43 | CI workflow concurrency: stop main churn cancelling dispatch/schedule runs | One-line group expression: workflow_dispatch/schedule events now get a per-run concurrency group (github.run_id) while push/PR keep the shared ref group with cancel-in-progress — fixes the release-browser-matrix livelock (cancelled twice on 2026-07-19/20 by main merges mid-run; the weekly Sunday 18:00 UTC scheduled run was subject to the same cancellation). release-browser-matrix is not a required branch-protection check; pin/scope checkers do not constrain the concurrency block. Accepted side effect: deliberate runs can overlap push runs. Rollback: plain revert. | check:github-actions PASS; check:ci-scope PASS; format:check PASS (repo files; local-only .claude/settings.local.json warning is gitignored) | +| 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: #901 rerank regression closeout) | fd5dcd1582e1c7aa8bf92ff097575e0c68aa930d | #901 rerank regression: baseline re-eval + offline guard net + saturated-tie fix | Baseline eval-canary dispatch on remediated main b3ae061 (run 29731533081) PASSED — golden retrieval 36/36 + answer-quality green, confirming #913-#926 closed the live 4/36 regression (July 19 failing run tested the raw #901 state; fixes landed 1-8h after it, never re-evaled until now). Residual defect found by new offline repro and fixed: with byte-identical imputed fast-path primaries, selection's clamped keys tie exactly and the chunk-id fallback decides, which secondStageScore's position adjustment launders into releaseRankScore (buried the CIWA answer doc in repro); fix = contentRankScore carried on RetrievalCandidate as tie-break between clamped rerankScore and chunk id, never added to scores. Amended #901's own never-live-validated saturated-tie pin to content-rank-then-id. Guards: tests/rag-fast-path-ordering.test.ts (4 end-to-end eval-shape repros) + ranking-tuning gates (4 golden-mapped hard negatives at production weights + full-snapshot high-risk). Fixture aliases NOT changed (Step C not triggered — identity cases pass live). | Targeted 4-suite vitest 37/37; npm run test 2981 passed (sole fail = container-only pdf-extraction-budget artifact); verify:cheap green through all 18 pre-test checks incl. lint+typecheck; check:production-readiness PASS on substantive checks (2 FAILs = documented missing-secret class in this container); build + client-bundle scan + check:rag:fixtures PASS | From c19715dcb18bb4b5696b1cab7b6736bbf49c635b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 10:13:50 +0000 Subject: [PATCH 3/3] test(rag): pin governance-conservative tie direction; correct guard-surface comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups from the pre-merge rag-retrieval and clinical-governance reviewer passes: acknowledge in-code that the tie-break key carries the bounded source-governance terms (deliberate — conservative direction at ties), correct the snapshot-gate caveat (the gate keys on relevanceGrade metrics, not documentMatch flags), state which repro binds the tie-break versus the remediation stack, and add an end-to-end case proving an outdated/poor-extraction twin loses the saturated tie to the current document instead of winning it by chunk id. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- src/lib/retrieval-selection.ts | 6 ++- tests/rag-fast-path-ordering.test.ts | 61 +++++++++++++++++++++++++++- tests/ranking-tuning.test.ts | 8 ++-- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 74ca25239..90aabfdbb 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -493,7 +493,11 @@ export function buildRetrievalCandidates( // Carried ONLY as the last tie-break before chunk id: on the embedding-free fast path, // imputed primaries make clamped score/lexical/rerank ties routine, and a chunk-id tie-break // is arbitrary — the second stage's position-based adjustment then launders that arbitrary - // winner into the released order. Never added to score. + // winner into the released order. Never added to score. Note this rankScore is the full + // clinical rank, which includes the bounded source-governance metadata terms — deliberate + // for a tie-break: among otherwise-indistinguishable candidates it prefers the current, + // well-extracted document (the conservative direction), unlike governance WEIGHTING of the + // primary score, which the NOTE below still forbids. contentRankScore: result.score_explanation?.rankScore ?? result.score_explanation?.preClampFinalScore ?? diff --git a/tests/rag-fast-path-ordering.test.ts b/tests/rag-fast-path-ordering.test.ts index 4c04783e7..a0d5cf007 100644 --- a/tests/rag-fast-path-ordering.test.ts +++ b/tests/rag-fast-path-ordering.test.ts @@ -12,7 +12,11 @@ import type { SearchResult } from "../src/lib/types"; // (rag-candidate-sources.ts: similarity = 0.62 + min(text_rank, 1) * 0.3), so unrelated // documents matching the same query terms collapse to byte-identical vector/text/hybrid // scores and only content-aware ranking can separate them. Each test replays one failed -// eval case's shape through the production ordering pipeline. +// eval case's shape through the production ordering pipeline. Guard surface: the CIWA +// case is the one whose top-1 assertion binds the selection contentRankScore tie-break +// (its candidates tie on every clamped key); the lithium/clozapine/safety-plan cases win +// earlier (subject boosts, table-type boosts, hybrid+id release order) and guard the +// remediation stack end-to-end rather than the tie-break itself. type QueryClass = ReturnType["queryClass"]; @@ -306,4 +310,59 @@ describe("text-fast-path ordering under imputed identical primaries", () => { // Both are legitimate sources — the policy document must stay retrievable, not be dropped. expect(released.map((result) => result.document_id)).toContain("rkpg-policy-doc"); }); + + it("prefers the current document over an outdated twin at exact saturated ties (conservative direction)", () => { + // The tie-break key is the full clinical rank, which includes the bounded source-governance + // terms — so among otherwise-identical candidates the current, well-extracted document wins. + // This pins the conservative direction of the governance component at ties. + const query = "What monitoring is required for lithium therapy?"; + const { queryClass } = classifyRagQuery(query); + const sharedFields = { + title: "Mood Stabiliser Guideline", + content: + "Serum lithium level should be checked 12 hours after the last dose during maintenance therapy, with renal function and thyroid function tests every six months.", + }; + const governanceMetadata = (overrides: Partial>) => ({ + source_title: null, + publisher: null, + jurisdiction: null, + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current" as const, + clinical_validation_status: "locally_reviewed" as const, + extraction_quality: "good" as const, + ...overrides, + }); + const currentDoc = imputedResult({ + ...sharedFields, + id: "guideline-current", + document_id: "current-doc", + file_name: "MoodStabiliserGuideline (SMHS).pdf", + source_metadata: governanceMetadata({}), + }); + const outdatedTwin = imputedResult({ + ...sharedFields, + id: "guideline-archived", + document_id: "archived-doc", + file_name: "MoodStabiliserGuideline 2019 (SMHS).pdf", + source_metadata: governanceMetadata({ document_status: "outdated", extraction_quality: "poor" }), + }); + + // Note "guideline-archived" < "guideline-current" lexicographically: without the + // governance-aware tie-break the chunk-id fallback would seat the outdated twin first. + const released = runTextFastPathOrdering({ + query, + queryClass, + candidates: [outdatedTwin, currentDoc], + topK: 8, + }); + + expect(released[0]?.document_id).toBe("current-doc"); + // Conservative availability: the outdated document is demoted, not dropped. + expect(released.map((result) => result.document_id)).toContain("archived-doc"); + }); }); diff --git a/tests/ranking-tuning.test.ts b/tests/ranking-tuning.test.ts index 9e0cac1cc..41ab53d10 100644 --- a/tests/ranking-tuning.test.ts +++ b/tests/ranking-tuning.test.ts @@ -128,9 +128,11 @@ describe("offline ranking tuner", () => { it("keeps the golden-regression hard negatives below the first relevant candidate at production weights", () => { // The four cases that failed the 2026-07-19 live golden retrieval eval on the raw #901 - // ordering (remediated same-day by #913-#926). Snapshot-builder caveat: labelMatches does - // not apply the eval's clinicalDocumentAliases, so alias-keyed documentMatch flags are - // false snapshot-wide — the content-graded relevant candidates carry this gate instead. + // ordering (remediated same-day by #913-#926). This gate keys on relevanceGrade-derived + // metrics (missing positives, hard-negative ordering), not on documentMatch flags, so the + // snapshot builder's alias-free labelMatches cannot weaken it. It exercises the snapshot + // proxy scorer at production weights — a complementary floor, not a test of the + // selectRetrievalEvidence comparator (tests/rag-fast-path-ordering.test.ts covers that). const goldenRegressionCaseIds = [ "lithium-therapy-monitoring", "clozapine-anc-threshold",