From fd3131803d1a6724ee924a52636773f545749f0f Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:38:12 +0800 Subject: [PATCH 1/4] fix: require subject evidence for monitoring ranges Prevent unrelated structured tables from satisfying medication monitoring threshold queries. Preserve the clinical subject through retrieval selection and document the remaining distinct RAG issues.\n\nVerified with focused Vitest, eval:rag:offline, verify:cheap, verify:pr-local, production-readiness, and a narrow live lithium diagnostic. --- docs/outstanding-issues.md | 2 ++ src/lib/rag/rag.ts | 20 ++++++++++++ src/lib/retrieval-selection.ts | 5 +-- tests/retrieval-query-variants.test.ts | 43 ++++++++++++++++++++++++++ tests/retrieval-selection.test.ts | 9 ++++++ 5 files changed, 77 insertions(+), 2 deletions(-) diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 63b00b7af..1566d8083 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -31,6 +31,8 @@ Durable, cross-session memory of everything still outstanding for this repo: ope ## Open items +> **RAG reconciliation correction (2026-07-23):** fresh current-main live evidence supersedes the broad diagnosis in #018. The three named misses are not one composer defect. Lithium was an unrelated-table retrieval fast-path defect and is being remediated separately by requiring co-located medication-subject evidence before accepting a structured-threshold fast path. ADHD still retrieves a relevant chart-heavy CAMHS source but exhausts the extractive route budget; metabolic retrieves the correct AKG source but selects schedule-free prose; #019 remains post-retrieval comparison source selection. Keep #029 open for the remaining fallback-stub cases. Do not combine these residuals or change ranking scores, comparator ordering, aliases, clamps, or semantic reranking without a separate reproducer and canary pair. + | ID | Pri | Type | Summary | Detail / next action | Source | Added | | ---- | --- | ----- | ------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | | #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 6f3836ed4..9f900196e 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -143,6 +143,7 @@ import { isMedicationDoseEvidenceQuery, medicationDoseEvidenceQueryIntent, medicationDoseQueryContext, + medicationMonitoringQuerySubjectTokens, normalizedClinicalSearchTokens, rankClinicalResults, } from "@/lib/clinical-search"; @@ -182,6 +183,7 @@ import { buildIndexingQuality, collectMemoryCards, deriveConfidence, + evidenceTextForGate, fallbackReasonFromRouting, isProviderGenerationDegraded, machineReadableFallbackAnswer, @@ -1923,6 +1925,24 @@ export function evaluateEvidenceCoverageGate( sourceImageSatisfied, }; } + const monitoringSubjectTokens = medicationMonitoringQuerySubjectTokens(query); + const hasStructuredThresholdSubject = + monitoringSubjectTokens.length === 0 || + top.some((result) => { + if (!hasStructuredThresholdEvidence(result)) return false; + const resultTokens = new Set(normalizedClinicalSearchTokens(evidenceTextForGate(result))); + const matchedSubjectTokens = monitoringSubjectTokens.filter((token) => resultTokens.has(token)).length; + return matchedSubjectTokens >= Math.min(2, monitoringSubjectTokens.length); + }); + if (hasStructuredThreshold && !hasStructuredThresholdSubject) { + return { + accepted: false, + reason: "missing_structured_threshold_subject_evidence", + strategy: "text_fast_path", + sourceImageRequired, + sourceImageSatisfied, + }; + } return { accepted: hasStructuredThreshold && strongestScore >= 0.58, reason: hasStructuredThreshold ? "structured_threshold_evidence_gate" : "missing_structured_threshold_evidence", diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 816721150..6d3ab2ecb 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -372,7 +372,8 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): medicationEvidenceIntent.asksAmount || medicationEvidenceIntent.asksRoute || medicationEvidenceIntent.asksFrequency; const asksDoseAmount = medicationEvidenceIntent.asksAmount; const asksMedicationMonitoring = - queryClass === "medication_dose_risk" && /\b(?:monitor\w*|baseline|blood|level)\b/.test(normalizedQuery); + (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && + /\b(?:monitor\w*|baseline|blood|level)\b/.test(normalizedQuery); const clinicalSubjectTokens = asksMedicationMonitoring ? medicationMonitoringQuerySubjectTokens(query) : medicationDoseQuerySubjectTokens(query); @@ -420,7 +421,7 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): if (medicationEvidenceIntent.asksRoute) requiredTermSignals.push("route"); } if ( - queryClass === "medication_dose_risk" && + (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && (asksDoseRoute || asksMedicationMonitoring) && clinicalSubjectTokens.length > 0 ) { diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index a0055bc52..84699d8cc 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -719,6 +719,49 @@ describe("retrieval query variants", () => { ).toMatchObject({ accepted: true, reason: "clozapine_blood_action_structured_threshold" }); }); + it("requires medication-subject evidence on a structured monitoring threshold fast gate", () => { + const query = "What lithium level range is used for maintenance monitoring?"; + + expect( + evaluateEvidenceCoverageGate( + query, + [ + result({ + title: "Clozapine monitoring table", + content: "Structured monitoring threshold table.", + similarity: 0.82, + table_facts: [ + tableFact({ clinical_parameter: "ANC", threshold_value: "< 1.5", action: "Withhold clozapine." }), + ], + }), + ], + "table_threshold", + ), + ).toMatchObject({ accepted: false, reason: "missing_structured_threshold_subject_evidence" }); + + expect( + evaluateEvidenceCoverageGate( + query, + [ + result({ + title: "Lithium monitoring", + content: "Lithium maintenance monitoring range.", + similarity: 0.72, + table_facts: [ + tableFact({ + table_title: "Lithium level monitoring", + clinical_parameter: "Maintenance range", + threshold_value: "0.6-0.8 mmol/L", + action: "Review the lithium level.", + }), + ], + }), + ], + "table_threshold", + ), + ).toMatchObject({ accepted: true, reason: "structured_threshold_evidence_gate" }); + }); + it("requires both route and numeric dose evidence for dose-route fast gates", () => { expect( evaluateEvidenceCoverageGate( diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index febf313ca..75236b9f4 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -309,6 +309,15 @@ describe("retrieval source selection", () => { ); }); + it("keeps the medication subject required for monitoring range retrieval", () => { + const intent = buildRetrievalIntent( + "What lithium level range is used for maintenance monitoring?", + "table_threshold", + ); + + expect(intent.requiredTermSignals).toContain("clinical_subject"); + }); + it("promotes exact source-table image evidence for clozapine ANC visual requests", () => { const selection = selectRetrievalEvidence({ query: "Show the source table image for the clozapine ANC monitoring table.", From 9236ce981390baa8ab8ec1fbd78e6714aff85a6b Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:58:16 +0800 Subject: [PATCH 2/4] Revert "fix: require subject evidence for monitoring ranges" This reverts commit fd3131803d1a6724ee924a52636773f545749f0f. --- docs/outstanding-issues.md | 2 -- src/lib/rag/rag.ts | 20 ------------ src/lib/retrieval-selection.ts | 5 ++- tests/retrieval-query-variants.test.ts | 43 -------------------------- tests/retrieval-selection.test.ts | 9 ------ 5 files changed, 2 insertions(+), 77 deletions(-) diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 1566d8083..63b00b7af 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -31,8 +31,6 @@ Durable, cross-session memory of everything still outstanding for this repo: ope ## Open items -> **RAG reconciliation correction (2026-07-23):** fresh current-main live evidence supersedes the broad diagnosis in #018. The three named misses are not one composer defect. Lithium was an unrelated-table retrieval fast-path defect and is being remediated separately by requiring co-located medication-subject evidence before accepting a structured-threshold fast path. ADHD still retrieves a relevant chart-heavy CAMHS source but exhausts the extractive route budget; metabolic retrieves the correct AKG source but selects schedule-free prose; #019 remains post-retrieval comparison source selection. Keep #029 open for the remaining fallback-stub cases. Do not combine these residuals or change ranking scores, comparator ordering, aliases, clamps, or semantic reranking without a separate reproducer and canary pair. - | ID | Pri | Type | Summary | Detail / next action | Source | Added | | ---- | --- | ----- | ------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | | #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 9f900196e..6f3836ed4 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -143,7 +143,6 @@ import { isMedicationDoseEvidenceQuery, medicationDoseEvidenceQueryIntent, medicationDoseQueryContext, - medicationMonitoringQuerySubjectTokens, normalizedClinicalSearchTokens, rankClinicalResults, } from "@/lib/clinical-search"; @@ -183,7 +182,6 @@ import { buildIndexingQuality, collectMemoryCards, deriveConfidence, - evidenceTextForGate, fallbackReasonFromRouting, isProviderGenerationDegraded, machineReadableFallbackAnswer, @@ -1925,24 +1923,6 @@ export function evaluateEvidenceCoverageGate( sourceImageSatisfied, }; } - const monitoringSubjectTokens = medicationMonitoringQuerySubjectTokens(query); - const hasStructuredThresholdSubject = - monitoringSubjectTokens.length === 0 || - top.some((result) => { - if (!hasStructuredThresholdEvidence(result)) return false; - const resultTokens = new Set(normalizedClinicalSearchTokens(evidenceTextForGate(result))); - const matchedSubjectTokens = monitoringSubjectTokens.filter((token) => resultTokens.has(token)).length; - return matchedSubjectTokens >= Math.min(2, monitoringSubjectTokens.length); - }); - if (hasStructuredThreshold && !hasStructuredThresholdSubject) { - return { - accepted: false, - reason: "missing_structured_threshold_subject_evidence", - strategy: "text_fast_path", - sourceImageRequired, - sourceImageSatisfied, - }; - } return { accepted: hasStructuredThreshold && strongestScore >= 0.58, reason: hasStructuredThreshold ? "structured_threshold_evidence_gate" : "missing_structured_threshold_evidence", diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 6d3ab2ecb..816721150 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -372,8 +372,7 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): medicationEvidenceIntent.asksAmount || medicationEvidenceIntent.asksRoute || medicationEvidenceIntent.asksFrequency; const asksDoseAmount = medicationEvidenceIntent.asksAmount; const asksMedicationMonitoring = - (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && - /\b(?:monitor\w*|baseline|blood|level)\b/.test(normalizedQuery); + queryClass === "medication_dose_risk" && /\b(?:monitor\w*|baseline|blood|level)\b/.test(normalizedQuery); const clinicalSubjectTokens = asksMedicationMonitoring ? medicationMonitoringQuerySubjectTokens(query) : medicationDoseQuerySubjectTokens(query); @@ -421,7 +420,7 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): if (medicationEvidenceIntent.asksRoute) requiredTermSignals.push("route"); } if ( - (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && + queryClass === "medication_dose_risk" && (asksDoseRoute || asksMedicationMonitoring) && clinicalSubjectTokens.length > 0 ) { diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index 84699d8cc..a0055bc52 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -719,49 +719,6 @@ describe("retrieval query variants", () => { ).toMatchObject({ accepted: true, reason: "clozapine_blood_action_structured_threshold" }); }); - it("requires medication-subject evidence on a structured monitoring threshold fast gate", () => { - const query = "What lithium level range is used for maintenance monitoring?"; - - expect( - evaluateEvidenceCoverageGate( - query, - [ - result({ - title: "Clozapine monitoring table", - content: "Structured monitoring threshold table.", - similarity: 0.82, - table_facts: [ - tableFact({ clinical_parameter: "ANC", threshold_value: "< 1.5", action: "Withhold clozapine." }), - ], - }), - ], - "table_threshold", - ), - ).toMatchObject({ accepted: false, reason: "missing_structured_threshold_subject_evidence" }); - - expect( - evaluateEvidenceCoverageGate( - query, - [ - result({ - title: "Lithium monitoring", - content: "Lithium maintenance monitoring range.", - similarity: 0.72, - table_facts: [ - tableFact({ - table_title: "Lithium level monitoring", - clinical_parameter: "Maintenance range", - threshold_value: "0.6-0.8 mmol/L", - action: "Review the lithium level.", - }), - ], - }), - ], - "table_threshold", - ), - ).toMatchObject({ accepted: true, reason: "structured_threshold_evidence_gate" }); - }); - it("requires both route and numeric dose evidence for dose-route fast gates", () => { expect( evaluateEvidenceCoverageGate( diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index 75236b9f4..febf313ca 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -309,15 +309,6 @@ describe("retrieval source selection", () => { ); }); - it("keeps the medication subject required for monitoring range retrieval", () => { - const intent = buildRetrievalIntent( - "What lithium level range is used for maintenance monitoring?", - "table_threshold", - ); - - expect(intent.requiredTermSignals).toContain("clinical_subject"); - }); - it("promotes exact source-table image evidence for clozapine ANC visual requests", () => { const selection = selectRetrievalEvidence({ query: "Show the source table image for the clozapine ANC monitoring table.", From 369414187a014ed97768a218030f8833be5ac2a8 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:19:56 +0800 Subject: [PATCH 3/4] docs: record rejected RAG canary findings Correct the monitoring-miss diagnosis and capture the two distinct answer-gate failures observed on byte-identical trees. Preserve the rejected lithium candidate as an archive-only follow-up and explicitly stop immediate live retries. --- docs/outstanding-issues.md | 79 ++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 63b00b7af..ad0fac530 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -27,47 +27,50 @@ Durable, cross-session memory of everything still outstanding for this repo: ope - Resolving an item moves its row to **Resolved / archive** with the date and a one-line outcome — rows are archived, not deleted, so the history stays auditable. - + ## Open items -| ID | Pri | Type | Summary | Detail / next action | Source | Added | -| ---- | --- | ----- | ------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | -| #005 | P3 | rec | `finalScore` saturates at clamp ceiling | Base + ~40 stacked boosts routinely exceed 1.0, so strong matches tie at 1.0 and order by an arbitrary `document_id` tiebreak. If ranking is ever revisited, break ties by the **pre-clamp** score rather than raising the `[0,1]` ceiling (downstream gates assume `[0,1]`). Ordering already sorts by the unbounded pre-clamp `rankScore` (`clinical-search.ts:1735,1927,1950-1955`), so the clamp confines only the reported confidence value, not result order. Not a defect on the current golden set; any change here is a protected RAG surface (canary required). | `docs/rag-hybrid-findings-and-todo.md` P1 item 4; `src/lib/clinical-search.ts:1735` | 2026-07-21 | -| #007 | P3 | rec | `/tools` vs `/?mode=tools` parallel Tools entry points | `/tools` (standalone `ApplicationsLauncherPage`) has no inbound in-app link; the sidebar Tools item uses `/?mode=tools`. Decide the canonical entry point and wire nav consistently, or drop the standalone `/tools` page + `/applications` redirect. Currently allowlisted in `tests/route-reachability.test.ts`. | `src/app/tools/page.tsx`; `src/app/applications/route.ts` | 2026-07-21 | -| #009 | P3 | rec | Confirm `/api/jobs` is intentionally server/ops-only | No client `fetch()` reaches `/api/jobs` (only tests import it). Confirm it is a deliberate ops/manual surface; if abandoned, remove it. | `src/app/api/jobs/route.ts` | 2026-07-21 | -| #010 | P3 | task | Un-built "Coming soon" controls across forms/favourites | ~10 disabled placeholders (forms refine/reset, favourites sort/add/new-set, move-to-set, remove-favourite). Correctly flagged (`aria-disabled` + "Coming soon"), not defects — wire when the underlying features land. | `forms-search-results-page.tsx`; `favourites-hub.tsx`; `favourites-command-library-page.tsx` | 2026-07-21 | -| #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | -| #012 | P3 | rec | Slim the lazy cross-mode differentials chunk | `cross-mode-differentials.ts` is dynamically imported (correctly code-split **out** of the initial/dashboard bundle — verified), but it pulls the full ~860 KB differentials snapshot (~125 KB gzip lazy chunk) just to build a tiny `{slug,title,clinicalHinge}` + presentations + aliases catalog. A precomputed lightweight index (generator + drift check, like the `specifiers-content` split / medications `fields=index`) would cut that lazy chunk ~5–10×. Not a bundle leak — an M-effort slim. | `src/lib/cross-mode-differentials.ts`; `src/components/clinical-dashboard/cross-mode-links.tsx:150`; session 2026-07-21 (build:analyze) | 2026-07-21 | -| #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | -| #014 | P3 | rec | Realize the `next/image` win on signed previews | `next.config` `images` (AVIF + `*.supabase.co` `remotePatterns` pinned to the project host, from #1024) is currently inert — signed document/image previews still render as raw ``. Route them through `next/image` to actually get AVIF + lazy optimization. | #1024; `src/components/clinical-dashboard/signed-image.tsx`; session 2026-07-21 | 2026-07-21 | -| #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `ƒ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them). | session 2026-07-21 (build route table + design audit) | 2026-07-21 | -| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. | session 2026-07-21 (measurement pass) | 2026-07-21 | -| #018 | P2 | task | Targeting misses are answer-composition, NOT retrieval-depth | DIAGNOSED 2026-07-22 from the run #61 --dump-answers artifact: retrieval is healthy (grounded=true, 4-6 citations, high_confidence_extractive_retrieval). lithium + adhd return the source_backed_review_fallback boilerplate stub ("a full written answer could not be completed just now") with answer_sections: []; metabolic returns a mangled 73-char fragment ("The guidance for metabolic is that compliance, monitoring and evaluation."). NOT corpus-content, NOT retrieval-depth. Root cause is the extractive composer (src/lib/rag/rag-extractive-answer.ts). Do NOT widen the eval matcher (rag-eval-cases.ts:192) — the stub contains monitoring/maintenance/level/range, so widening would make stubs falsely PASS. Fix is a protected-surface answer-behaviour change (provider-backed answer-quality eval + approval). | run #61 dump artifact; session 2026-07-22 | 2026-07-21 | -| #019 | P2 | task | Admission doc dropped by answer-stage re-rank, not retrieval | DIAGNOSED 2026-07-22 from run #61 artifacts: retrieval coverage is PERFECT — golden documentRecallAt5=1, reciprocalRankAt10=1, missingDocumentSubstrings=[], with "Admission of Community Patients (AKG).pdf" at raw rank 4-5 (matches the strict alias AdmissionCommunityPts). The answer stage then DROPS it from answer.sources top-5 and promotes "Falls Prevention and Management (AKG).pdf" (raw rank 8, off-topic) in its place. So the defect is post-retrieval evidence balancing (cross-document-synthesis.ts balanceCrossDocumentResults / rankAnswerEvidence), NOT comparison-class retrieval coverage. Separately: nothing guarantees BOTH sides of a comparison are represented — comparison_multi_document_gate (rag.ts:2043) is satisfied by any 2 distinct docs. Fix = protected surface (canary pair + ~$2-4 approval). | run #61 artifacts; session 2026-07-22 | 2026-07-21 | -| #020 | P3 | task | Validate eval:quality cost readout post-fix | The estimator fix landed in PR #1050 (zero-usage cases count $0 with rates configured; provider-attempted cases with missing usage stay n/a). Remaining action: confirm the next canary run prints a real dollar figure in Answer Metrics, then archive this row. | PR #1050; run #60/#61 Answer Metrics tables | 2026-07-21 | -| #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | -| #022 | P2 | task | Source-governance metadata refresh (operator) | **Worklist generated 2026-07-22 ($0, read-only): `docs/source-governance-refresh-worklist-2026-07-22.md`.** Reframed - this is NOT 59 clinical reviews. Of the 124 documents surfacing in canary top results, 59 are review-required, and **38 (64 pct) are the BMJ published-reference tier all sitting at `clinical_validation_status: unverified`** - one attestation-policy decision, not 38 reviews. The remaining 21 are genuine local WA health-service reviews (FSH 7, NMHS 4, CAMHS 3, AKG 2, KEMH 2, RPBG 2, RKPG 1), mostly `document_status: review_due`. Burn-down: top-10 documents clear 44 pct of flagged slots, top-20 clear 66 pct. Next: decide the BMJ attestation policy, then attest local docs by visibility (start `Clozapine Management by GP (NMHS)`, 22 slots at rank 1). | runs #61/#57 Source Governance data; `docs/source-governance-refresh-worklist-2026-07-22.md` | 2026-07-21 | -| #023 | P3 | task | Read Sunday 2026-07-26 scheduled-run artifacts | The 18:00 UTC scheduled runs deliver three free datapoints at once: first full-44 weekly canary (validates the #1044 ANSWER_CASE_LIMIT raise), browser-matrix flake second datapoint (webkit ui-route-coverage now reproduced + root-caused 2026-07-22 → see #024; firefox ui-formulation:91 still awaits a datapoint), and the irrelevant@10 labeling-audit artifact (§3.1 human-decision class). Read all three, then disposition. | sessions 2026-07-20/21; branch-review-ledger convergence notes | 2026-07-21 | -| #024 | P3 | issue | WebKit e2e `_rsc`-prefetch access-control-checks errors | verify:release:offline on `main` ce32fe170 (2026-07-22) reproduced #023's webkit clause: **6/6 deterministic** failures in `tests/ui-route-coverage.spec.ts` (Therapy Compass; DSM home/comparison; Specifier comparison/map; Differential stream), each a `pageerror … ?_rsc=… due to access control checks` on Next.js RSC prefetch — Chromium + Firefox clean. Not merge-blocking (required gate `test:e2e:pr` is chromium-only; the full webkit matrix is advisory/release-time). Most likely a Playwright route-interception × WebKit interaction, not a Safari user defect. Next: decide (a) allow/mock the `_rsc` routes for the `webkit` e2e project, or (b) confirm real Safari impact — before trusting the full-matrix webkit gate at release. NB the 2 other webkit fails (`ui-stress:412`, `ui-universal-search:210`) passed on isolated re-run = true flake. | session 2026-07-22 (verify:release:offline, `main` ce32fe170); refines #023 | 2026-07-22 | -| #025 | P2 | task | Activate the three webhooks (operator secrets) | Merged (#968) + deployed but inert — verified live: `POST /api/webhooks/railway` returns `503 webhook_not_configured`. To turn on: (1) Railway → set `RAILWAY_WEBHOOK_SECRET` + add the `?token=…` webhook URL; (2) the chat URLs `SLACK_WEBHOOK_URL`/`DISCORD_WEBHOOK_URL` must be set in BOTH places — the Railway **app/server env** (the receiver forwards deploy alerts via `postChatNotification`, which reads server env, so repo-secret-only leaves the Railway webhook authenticated but returning `delivered:false`) AND as **GitHub repo secrets** (the CI-failure workflow reads `secrets.*`); (3) `SUPABASE_INGESTION_WEBHOOK_SECRET`. Each fails closed until set, so this is pure ops. See docs/webhooks.md. | session 2026-07-22; PR #968; docs/webhooks.md | 2026-07-22 | -| #026 | P2 | task | Wire the Supabase document-change trigger | The `/api/webhooks/supabase/document-change` receiver is live but nothing calls it. The trigger (`notify_document_change_ingestion_webhook()` + `documents_ingestion_webhook`, SQL in docs/webhooks.md §3) must land as a **committed migration** — drift covers functions/triggers, so applying raw SQL to live would fail `check:drift`: add the migration, reconcile `supabase/schema.sql`, run `npm run drift:manifest` (needs a local Supabase container), then deploy. Operator-applied live state is the Vault secret (`select vault.create_secret('','ingestion_webhook_secret')`) + the base-URL GUC `app.ingestion_webhook_base_url`, which is MANDATORY per environment (the trigger no-ops when it is unset, so `reindex_requested` flips never enqueue — there is no production default). Trigger is `AFTER UPDATE` on `reindex_requested` only — NOT `INSERT` (an INSERT-fired webhook can beat the upload route's own job insert, which then deletes its just-created doc → intermittent upload failures); external inserts set `reindex_requested=true` to be picked up. Note delivery is at-most-once (pg_net has no retry) and there is NO built-in recovery (automatic or manual) for a dropped insert — neither the autopilot nor `scripts/reindex.ts` sweeps `queued` docs with no job (`hasIncompleteDocumentsWithoutOpenJobs` requires `queuedDocuments === 0`). External imports must insert then issue a SEPARATE UPDATE flipping `reindex_requested=true` (an insert with the flag pre-set won't fire the AFTER UPDATE trigger). The app upload path is NOT a guarantee either: it inserts the `documents` row then the `ingestion_jobs` row as two separate non-atomic calls (rollback only on a job-insert error), so a crash/timeout between them strands the same queued-without-job row nothing sweeps. The only true guarantees are an atomic enqueue (document + job in one transaction/RPC) or a scheduled queued-without-job sweep. Until applied, webhook #5 never fires. | session 2026-07-22; docs/webhooks.md §3 | 2026-07-22 | -| #027 | P3 | rec | External uptime monitor independent of GitHub/Railway | `live-domain-monitor.yml` runs on GitHub's cron, so it won't run in exactly the outage it should catch (Actions or the deploy itself down). Add an off-platform synthetic monitor (UptimeRobot / Better Stack / Checkly) hitting `/api/health` with a webhook alert. Provider setup, not code. | session 2026-07-22 webhook review | 2026-07-22 | -| #028 | P3 | rec | Runtime error tracking (Sentry or similar) | No error tracking in the repo — production exceptions on `psychiatry.tools`, including how often `RAG_PROVIDER_MODE=auto` silently degrades to source-only, are invisible. Weigh adding `@sentry/nextjs` (dependency + DSN secret + instrumentation) vs cost; alert → chat/issue. Provider-backed; needs explicit sign-off before adding the dependency. | session 2026-07-22 webhook review | 2026-07-22 | -| #029 | P2 | issue | 12 of 30 answer-quality cases return the fallback stub | run #61 --dump-answers: 12/30 quality cases emit the source_backed_review_fallback boilerplate with answer_sections: [], all grounded with 4-6 citations. Some still PASS targeting because the stub echoes query keywords (the contraindication/document_lookup matchers need only a keyword), so the targeting metric MASKS the problem for those intents. Superset of #018 — fix in the extractive composer, validate with the provider-backed answer eval. | run #61 dump artifact; session 2026-07-22 | 2026-07-22 | -| #030 | P3 | issue | Wide-tier alias lets one doc satisfy both comparison slots | In src/lib/eval-document-matching.ts, "Admission to Discharge for Mental Health Inpatients" appears in BOTH the AdmissionCommunityPts and Discharge alias lists, so a single document can satisfy both expectedFiles slots and make allHit true — a latent false-pass on admission-discharge cases. Not firing today (that doc is not in the failing top-5) but it would mask a real miss. Tighten the tables so one doc cannot fill both sides. | src/lib/eval-document-matching.ts:32-65; session 2026-07-22 | 2026-07-22 | -| #031 | P3 | issue | Canary Source Governance table reports all zeros | Run #61 `answer-quality.log` prints `## Source Governance` with `Top results \| 0` and every rate 0, even though the run's `golden-retrieval.json` `topResults` carry full per-result governance metadata (`document_status`, `clinical_validation_status`, `extraction_quality`). The operator-facing table is therefore not populated and the #022 worklist had to be derived from raw JSON. Fix the table's data wiring so governance is visible from the log itself. | run #61 eval-canary artifacts; session 2026-07-22 | 2026-07-22 | -| #032 | P3 | rec | Governance ranking weighting: REFUTED, not debt | The source-governance audit (PR #1051) flagged three "gaps": `review_due` carries no ranking penalty, `unknownCurrentnessPenalty` ships at 0, and `selectBestSourceRecommendation` ignores governance metadata. **These are deliberate, measured decisions — do NOT implement them as written.** Blanket metadata boosts/penalties in selection ordering were measured on 2026-07-02 to regress the golden retrieval eval to 16/23 (doc-recall@5 1.0→0.76, mrr 0.75→0.64). Two corpus facts make it unsafe: scores saturate at the clamp so stacked boosts fully override lexical relevance, and the corpus is only partially metadata-enriched while `normalizeSourceMetadata` coerces unenriched docs to `unknown`/`unverified` — so "unknown" ≠ "bad" and blanket weighting swings ranking approx. 0.35 for reasons unrelated to relevance. Even governance-as-tiebreak buried correct unenriched docs (3 designs bisected). Next action: none — treat as a guardrail. If ever revisited, RC8 (source-strength as a _filter_) is the tracked path, gated on `eval:retrieval:quality` 36/36 plus a live canary pair. | PR #118; `docs/rag-behaviour/refuted-approaches.md`; PR #1051 items 4/5/6 | 2026-07-22 | -| #033 | P3 | rec | Source governance metadata absent from the LLM prompt | `buildRagSourceBlock` omits `document_status`, `clinical_validation_status`, and `extraction_quality`, so the model cannot self-caveat during generation and governance is enforced only post-hoc. Generation-surface change: needs `eval:rag` plus `eval:quality --rag-only` (grounded-supported must not drop, citation-failure 0) and explicit approval. Carries the same "unknown ≠ bad" hazard as #032 — on a partially-enriched corpus the model would likely over-caveat correct sources, so design the prompt wording before spending an eval. | `src/lib/rag/rag-source-block.ts:126-198`; PR #1051 audit item 8 | 2026-07-22 | -| #034 | P3 | issue | Answer cache can serve stale governance metadata | `cacheIndexingVersion` derives the version from `updated_at` / `indexed_at` / `index_generation_id`, so a metadata-only `document_status` flip that bumps none of those is invisible to the passive guard. **Already mitigated**: every known status-write path calls `invalidateRagCachesForOwner` or `invalidateRagCachesForDocumentMutation`. Residual risk only — a future write path that omits the invalidator would serve stale governance until TTL. Next action: add a regression test pinning the invalidator call on status-mutating routes (cheaper and safer than touching the protected cache key). | `src/lib/rag/rag-cache.ts:382-438`; PR #1051 audit item 10 | 2026-07-22 | -| #035 | P3 | rec | Threshold-conflict detection covers only 3 params | `detectThresholdDisagreements` checks only ANC, WBC, and platelets paired with withholding verbs, so cross-source conflicts on medication doses, lithium/thyroid levels, or vital signs go undetected. Deliberately narrow (see the comment at `:469-474`). Broadening changes when an answer is classified `conflicting` and adds warnings — real false-positive risk. Needs new fixtures plus a behaviour review before any change. | `src/lib/evidence.ts:469-574`; PR #1051 audit item 7 | 2026-07-22 | -| #036 | P3 | rec | No explicit `is_public` visibility flag on documents | Public-corpus visibility is implicit: `owner_id IS NULL` on an `indexed` document (`resolveSearchScope`). The `metadata.public_corpus` marker is written by the promotion migrations but never used as a retrieval filter. Promotion is unconditional on `clinical_validation_status`, so unverified documents are publicly searchable — compensated by keeping `unverified_source` in the frontend-visible warning set. A hard schema flag touches RLS and the clinical-risk-gated retrieval RPCs; weigh against the existing compensating control before acting. | `supabase/schema.sql:61-108`; `src/lib/search-scope.ts:181-236`; PR #1051 audit item 3 | 2026-07-22 | -| #037 | P3 | rec | D5 trust-cap-all-claims flag parked OFF | `NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS` extends authority gating from high-risk claims to **all** supported claims (`deriveTrust`). Ships OFF by design; flipping it caps trust to `medium` for routine claims across the board — a product/clinical-UX decision, not a defect. Both states are test-pinned. Next action: product decision, then flip and re-baseline the UI expectations. | `src/lib/answer-render-policy.ts:159-177`; PR #1051 audit item 11 | 2026-07-22 | -| #038 | P3 | rec | Consolidate shared comparison behavior | Several clinical modes expose comparison workflows with similar selection, empty-state and mobile-dock needs. Define one shared behavioral contract before another comparison surface is added; keep mode-specific clinical content separate. This is a design-system recommendation, not a current defect. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #039 | P3 | rec | Consolidate catalogue toolbar patterns | Catalogue/search pages have independently evolved filter, sort, result-count and mobile toolbar behavior. Inventory the existing implementations and converge only the repeated interaction contract; do not flatten mode-specific search semantics. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #041 | P3 | rec | Extend the existing Factsheets reading model | Do not add a second patient-facing Factsheets mode. Future patient-content work should extend the existing Easy Read/Standard presentation and its accessibility/content contracts. Revisit only with a concrete user need and source-governance plan. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +> **RAG reconciliation correction (2026-07-23):** fresh current-main live evidence supersedes the broad diagnosis in #018. The three named misses are not one composer defect. Lithium reproduced an unrelated-table retrieval fast-path defect; ADHD still retrieves a relevant chart-heavy CAMHS source but exhausts the extractive route budget; metabolic retrieves the correct AKG source but selects schedule-free prose; #019 remains post-retrieval comparison source selection. A narrow lithium subject-evidence guard improved its targeting result from 0 to 1 with golden recall 1.0 and no reciprocal-rank regressions, but was reverted and rejected because the required full canary failed. Keep #029 open for the remaining fallback-stub cases. Do not combine these residuals or change ranking scores, comparator ordering, aliases, clamps, or semantic reranking without a separate reproducer and passing canary pair. + +| ID | Pri | Type | Summary | Detail / next action | Source | Added | +| ---- | --- | ----- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| #051 | P2 | task | Stabilise the live answer-quality canary before more RAG tuning | Two full canaries on different commits with byte-identical trees passed golden retrieval (document/content recall 1.0 and zero per-case reciprocal-rank regressions) but failed the answer gate for different reasons: candidate run `30007833352` exceeded unsupported-route p95 (6,227 ms > 4,000 ms), while reverted confirmation `30009207429` had citation-failure rate 0.0227 from the neuroleptic case selecting one wrong source. Do not spend on another immediate retry. First make the answer gate distinguish provider/latency variability from deterministic content regressions and add stable per-case diagnostics; then re-evaluate the archived lithium guard. The rejected commit is recoverable at local ref `refs/archive/rejected-rag/20260723/monitoring-subject-gate`. | Eval Canary runs `30007833352`, `30009207429`; PR #1093; session 2026-07-23 | 2026-07-23 | +| #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | +| #005 | P3 | rec | `finalScore` saturates at clamp ceiling | Base + ~40 stacked boosts routinely exceed 1.0, so strong matches tie at 1.0 and order by an arbitrary `document_id` tiebreak. If ranking is ever revisited, break ties by the **pre-clamp** score rather than raising the `[0,1]` ceiling (downstream gates assume `[0,1]`). Ordering already sorts by the unbounded pre-clamp `rankScore` (`clinical-search.ts:1735,1927,1950-1955`), so the clamp confines only the reported confidence value, not result order. Not a defect on the current golden set; any change here is a protected RAG surface (canary required). | `docs/rag-hybrid-findings-and-todo.md` P1 item 4; `src/lib/clinical-search.ts:1735` | 2026-07-21 | +| #007 | P3 | rec | `/tools` vs `/?mode=tools` parallel Tools entry points | `/tools` (standalone `ApplicationsLauncherPage`) has no inbound in-app link; the sidebar Tools item uses `/?mode=tools`. Decide the canonical entry point and wire nav consistently, or drop the standalone `/tools` page + `/applications` redirect. Currently allowlisted in `tests/route-reachability.test.ts`. | `src/app/tools/page.tsx`; `src/app/applications/route.ts` | 2026-07-21 | +| #009 | P3 | rec | Confirm `/api/jobs` is intentionally server/ops-only | No client `fetch()` reaches `/api/jobs` (only tests import it). Confirm it is a deliberate ops/manual surface; if abandoned, remove it. | `src/app/api/jobs/route.ts` | 2026-07-21 | +| #010 | P3 | task | Un-built "Coming soon" controls across forms/favourites | ~10 disabled placeholders (forms refine/reset, favourites sort/add/new-set, move-to-set, remove-favourite). Correctly flagged (`aria-disabled` + "Coming soon"), not defects — wire when the underlying features land. | `forms-search-results-page.tsx`; `favourites-hub.tsx`; `favourites-command-library-page.tsx` | 2026-07-21 | +| #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | +| #012 | P3 | rec | Slim the lazy cross-mode differentials chunk | `cross-mode-differentials.ts` is dynamically imported (correctly code-split **out** of the initial/dashboard bundle — verified), but it pulls the full ~860 KB differentials snapshot (~125 KB gzip lazy chunk) just to build a tiny `{slug,title,clinicalHinge}` + presentations + aliases catalog. A precomputed lightweight index (generator + drift check, like the `specifiers-content` split / medications `fields=index`) would cut that lazy chunk ~5–10×. Not a bundle leak — an M-effort slim. | `src/lib/cross-mode-differentials.ts`; `src/components/clinical-dashboard/cross-mode-links.tsx:150`; session 2026-07-21 (build:analyze) | 2026-07-21 | +| #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | +| #014 | P3 | rec | Realize the `next/image` win on signed previews | `next.config` `images` (AVIF + `*.supabase.co` `remotePatterns` pinned to the project host, from #1024) is currently inert — signed document/image previews still render as raw ``. Route them through `next/image` to actually get AVIF + lazy optimization. | #1024; `src/components/clinical-dashboard/signed-image.tsx`; session 2026-07-21 | 2026-07-21 | +| #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `ƒ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them). | session 2026-07-21 (build route table + design audit) | 2026-07-21 | +| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. | session 2026-07-21 (measurement pass) | 2026-07-21 | +| #018 | P2 | task | Targeting misses are answer-composition, NOT retrieval-depth | DIAGNOSED 2026-07-22 from the run #61 --dump-answers artifact: retrieval is healthy (grounded=true, 4-6 citations, high_confidence_extractive_retrieval). lithium + adhd return the source_backed_review_fallback boilerplate stub ("a full written answer could not be completed just now") with answer_sections: []; metabolic returns a mangled 73-char fragment ("The guidance for metabolic is that compliance, monitoring and evaluation."). NOT corpus-content, NOT retrieval-depth. Root cause is the extractive composer (src/lib/rag/rag-extractive-answer.ts). Do NOT widen the eval matcher (rag-eval-cases.ts:192) — the stub contains monitoring/maintenance/level/range, so widening would make stubs falsely PASS. Fix is a protected-surface answer-behaviour change (provider-backed answer-quality eval + approval). | run #61 dump artifact; session 2026-07-22 | 2026-07-21 | +| #019 | P2 | task | Admission doc dropped by answer-stage re-rank, not retrieval | DIAGNOSED 2026-07-22 from run #61 artifacts: retrieval coverage is PERFECT — golden documentRecallAt5=1, reciprocalRankAt10=1, missingDocumentSubstrings=[], with "Admission of Community Patients (AKG).pdf" at raw rank 4-5 (matches the strict alias AdmissionCommunityPts). The answer stage then DROPS it from answer.sources top-5 and promotes "Falls Prevention and Management (AKG).pdf" (raw rank 8, off-topic) in its place. So the defect is post-retrieval evidence balancing (cross-document-synthesis.ts balanceCrossDocumentResults / rankAnswerEvidence), NOT comparison-class retrieval coverage. Separately: nothing guarantees BOTH sides of a comparison are represented — comparison_multi_document_gate (rag.ts:2043) is satisfied by any 2 distinct docs. Fix = protected surface (canary pair + ~$2-4 approval). | run #61 artifacts; session 2026-07-22 | 2026-07-21 | +| #020 | P3 | task | Validate eval:quality cost readout post-fix | The estimator fix landed in PR #1050 (zero-usage cases count $0 with rates configured; provider-attempted cases with missing usage stay n/a). Remaining action: confirm the next canary run prints a real dollar figure in Answer Metrics, then archive this row. | PR #1050; run #60/#61 Answer Metrics tables | 2026-07-21 | +| #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | +| #022 | P2 | task | Source-governance metadata refresh (operator) | **Worklist generated 2026-07-22 ($0, read-only): `docs/source-governance-refresh-worklist-2026-07-22.md`.** Reframed - this is NOT 59 clinical reviews. Of the 124 documents surfacing in canary top results, 59 are review-required, and **38 (64 pct) are the BMJ published-reference tier all sitting at `clinical_validation_status: unverified`** - one attestation-policy decision, not 38 reviews. The remaining 21 are genuine local WA health-service reviews (FSH 7, NMHS 4, CAMHS 3, AKG 2, KEMH 2, RPBG 2, RKPG 1), mostly `document_status: review_due`. Burn-down: top-10 documents clear 44 pct of flagged slots, top-20 clear 66 pct. Next: decide the BMJ attestation policy, then attest local docs by visibility (start `Clozapine Management by GP (NMHS)`, 22 slots at rank 1). | runs #61/#57 Source Governance data; `docs/source-governance-refresh-worklist-2026-07-22.md` | 2026-07-21 | +| #023 | P3 | task | Read Sunday 2026-07-26 scheduled-run artifacts | The 18:00 UTC scheduled runs deliver three free datapoints at once: first full-44 weekly canary (validates the #1044 ANSWER_CASE_LIMIT raise), browser-matrix flake second datapoint (webkit ui-route-coverage now reproduced + root-caused 2026-07-22 → see #024; firefox ui-formulation:91 still awaits a datapoint), and the irrelevant@10 labeling-audit artifact (§3.1 human-decision class). Read all three, then disposition. | sessions 2026-07-20/21; branch-review-ledger convergence notes | 2026-07-21 | +| #024 | P3 | issue | WebKit e2e `_rsc`-prefetch access-control-checks errors | verify:release:offline on `main` ce32fe170 (2026-07-22) reproduced #023's webkit clause: **6/6 deterministic** failures in `tests/ui-route-coverage.spec.ts` (Therapy Compass; DSM home/comparison; Specifier comparison/map; Differential stream), each a `pageerror … ?_rsc=… due to access control checks` on Next.js RSC prefetch — Chromium + Firefox clean. Not merge-blocking (required gate `test:e2e:pr` is chromium-only; the full webkit matrix is advisory/release-time). Most likely a Playwright route-interception × WebKit interaction, not a Safari user defect. Next: decide (a) allow/mock the `_rsc` routes for the `webkit` e2e project, or (b) confirm real Safari impact — before trusting the full-matrix webkit gate at release. NB the 2 other webkit fails (`ui-stress:412`, `ui-universal-search:210`) passed on isolated re-run = true flake. | session 2026-07-22 (verify:release:offline, `main` ce32fe170); refines #023 | 2026-07-22 | +| #025 | P2 | task | Activate the three webhooks (operator secrets) | Merged (#968) + deployed but inert — verified live: `POST /api/webhooks/railway` returns `503 webhook_not_configured`. To turn on: (1) Railway → set `RAILWAY_WEBHOOK_SECRET` + add the `?token=…` webhook URL; (2) the chat URLs `SLACK_WEBHOOK_URL`/`DISCORD_WEBHOOK_URL` must be set in BOTH places — the Railway **app/server env** (the receiver forwards deploy alerts via `postChatNotification`, which reads server env, so repo-secret-only leaves the Railway webhook authenticated but returning `delivered:false`) AND as **GitHub repo secrets** (the CI-failure workflow reads `secrets.*`); (3) `SUPABASE_INGESTION_WEBHOOK_SECRET`. Each fails closed until set, so this is pure ops. See docs/webhooks.md. | session 2026-07-22; PR #968; docs/webhooks.md | 2026-07-22 | +| #026 | P2 | task | Wire the Supabase document-change trigger | The `/api/webhooks/supabase/document-change` receiver is live but nothing calls it. The trigger (`notify_document_change_ingestion_webhook()` + `documents_ingestion_webhook`, SQL in docs/webhooks.md §3) must land as a **committed migration** — drift covers functions/triggers, so applying raw SQL to live would fail `check:drift`: add the migration, reconcile `supabase/schema.sql`, run `npm run drift:manifest` (needs a local Supabase container), then deploy. Operator-applied live state is the Vault secret (`select vault.create_secret('','ingestion_webhook_secret')`) + the base-URL GUC `app.ingestion_webhook_base_url`, which is MANDATORY per environment (the trigger no-ops when it is unset, so `reindex_requested` flips never enqueue — there is no production default). Trigger is `AFTER UPDATE` on `reindex_requested` only — NOT `INSERT` (an INSERT-fired webhook can beat the upload route's own job insert, which then deletes its just-created doc → intermittent upload failures); external inserts set `reindex_requested=true` to be picked up. Note delivery is at-most-once (pg_net has no retry) and there is NO built-in recovery (automatic or manual) for a dropped insert — neither the autopilot nor `scripts/reindex.ts` sweeps `queued` docs with no job (`hasIncompleteDocumentsWithoutOpenJobs` requires `queuedDocuments === 0`). External imports must insert then issue a SEPARATE UPDATE flipping `reindex_requested=true` (an insert with the flag pre-set won't fire the AFTER UPDATE trigger). The app upload path is NOT a guarantee either: it inserts the `documents` row then the `ingestion_jobs` row as two separate non-atomic calls (rollback only on a job-insert error), so a crash/timeout between them strands the same queued-without-job row nothing sweeps. The only true guarantees are an atomic enqueue (document + job in one transaction/RPC) or a scheduled queued-without-job sweep. Until applied, webhook #5 never fires. | session 2026-07-22; docs/webhooks.md §3 | 2026-07-22 | +| #027 | P3 | rec | External uptime monitor independent of GitHub/Railway | `live-domain-monitor.yml` runs on GitHub's cron, so it won't run in exactly the outage it should catch (Actions or the deploy itself down). Add an off-platform synthetic monitor (UptimeRobot / Better Stack / Checkly) hitting `/api/health` with a webhook alert. Provider setup, not code. | session 2026-07-22 webhook review | 2026-07-22 | +| #028 | P3 | rec | Runtime error tracking (Sentry or similar) | No error tracking in the repo — production exceptions on `psychiatry.tools`, including how often `RAG_PROVIDER_MODE=auto` silently degrades to source-only, are invisible. Weigh adding `@sentry/nextjs` (dependency + DSN secret + instrumentation) vs cost; alert → chat/issue. Provider-backed; needs explicit sign-off before adding the dependency. | session 2026-07-22 webhook review | 2026-07-22 | +| #029 | P2 | issue | 12 of 30 answer-quality cases return the fallback stub | run #61 --dump-answers: 12/30 quality cases emit the source_backed_review_fallback boilerplate with answer_sections: [], all grounded with 4-6 citations. Some still PASS targeting because the stub echoes query keywords (the contraindication/document_lookup matchers need only a keyword), so the targeting metric MASKS the problem for those intents. Superset of #018 — fix in the extractive composer, validate with the provider-backed answer eval. | run #61 dump artifact; session 2026-07-22 | 2026-07-22 | +| #030 | P3 | issue | Wide-tier alias lets one doc satisfy both comparison slots | In src/lib/eval-document-matching.ts, "Admission to Discharge for Mental Health Inpatients" appears in BOTH the AdmissionCommunityPts and Discharge alias lists, so a single document can satisfy both expectedFiles slots and make allHit true — a latent false-pass on admission-discharge cases. Not firing today (that doc is not in the failing top-5) but it would mask a real miss. Tighten the tables so one doc cannot fill both sides. | src/lib/eval-document-matching.ts:32-65; session 2026-07-22 | 2026-07-22 | +| #031 | P3 | issue | Canary Source Governance table reports all zeros | Run #61 `answer-quality.log` prints `## Source Governance` with `Top results \| 0` and every rate 0, even though the run's `golden-retrieval.json` `topResults` carry full per-result governance metadata (`document_status`, `clinical_validation_status`, `extraction_quality`). The operator-facing table is therefore not populated and the #022 worklist had to be derived from raw JSON. Fix the table's data wiring so governance is visible from the log itself. | run #61 eval-canary artifacts; session 2026-07-22 | 2026-07-22 | +| #032 | P3 | rec | Governance ranking weighting: REFUTED, not debt | The source-governance audit (PR #1051) flagged three "gaps": `review_due` carries no ranking penalty, `unknownCurrentnessPenalty` ships at 0, and `selectBestSourceRecommendation` ignores governance metadata. **These are deliberate, measured decisions — do NOT implement them as written.** Blanket metadata boosts/penalties in selection ordering were measured on 2026-07-02 to regress the golden retrieval eval to 16/23 (doc-recall@5 1.0→0.76, mrr 0.75→0.64). Two corpus facts make it unsafe: scores saturate at the clamp so stacked boosts fully override lexical relevance, and the corpus is only partially metadata-enriched while `normalizeSourceMetadata` coerces unenriched docs to `unknown`/`unverified` — so "unknown" ≠ "bad" and blanket weighting swings ranking approx. 0.35 for reasons unrelated to relevance. Even governance-as-tiebreak buried correct unenriched docs (3 designs bisected). Next action: none — treat as a guardrail. If ever revisited, RC8 (source-strength as a _filter_) is the tracked path, gated on `eval:retrieval:quality` 36/36 plus a live canary pair. | PR #118; `docs/rag-behaviour/refuted-approaches.md`; PR #1051 items 4/5/6 | 2026-07-22 | +| #033 | P3 | rec | Source governance metadata absent from the LLM prompt | `buildRagSourceBlock` omits `document_status`, `clinical_validation_status`, and `extraction_quality`, so the model cannot self-caveat during generation and governance is enforced only post-hoc. Generation-surface change: needs `eval:rag` plus `eval:quality --rag-only` (grounded-supported must not drop, citation-failure 0) and explicit approval. Carries the same "unknown ≠ bad" hazard as #032 — on a partially-enriched corpus the model would likely over-caveat correct sources, so design the prompt wording before spending an eval. | `src/lib/rag/rag-source-block.ts:126-198`; PR #1051 audit item 8 | 2026-07-22 | +| #034 | P3 | issue | Answer cache can serve stale governance metadata | `cacheIndexingVersion` derives the version from `updated_at` / `indexed_at` / `index_generation_id`, so a metadata-only `document_status` flip that bumps none of those is invisible to the passive guard. **Already mitigated**: every known status-write path calls `invalidateRagCachesForOwner` or `invalidateRagCachesForDocumentMutation`. Residual risk only — a future write path that omits the invalidator would serve stale governance until TTL. Next action: add a regression test pinning the invalidator call on status-mutating routes (cheaper and safer than touching the protected cache key). | `src/lib/rag/rag-cache.ts:382-438`; PR #1051 audit item 10 | 2026-07-22 | +| #035 | P3 | rec | Threshold-conflict detection covers only 3 params | `detectThresholdDisagreements` checks only ANC, WBC, and platelets paired with withholding verbs, so cross-source conflicts on medication doses, lithium/thyroid levels, or vital signs go undetected. Deliberately narrow (see the comment at `:469-474`). Broadening changes when an answer is classified `conflicting` and adds warnings — real false-positive risk. Needs new fixtures plus a behaviour review before any change. | `src/lib/evidence.ts:469-574`; PR #1051 audit item 7 | 2026-07-22 | +| #036 | P3 | rec | No explicit `is_public` visibility flag on documents | Public-corpus visibility is implicit: `owner_id IS NULL` on an `indexed` document (`resolveSearchScope`). The `metadata.public_corpus` marker is written by the promotion migrations but never used as a retrieval filter. Promotion is unconditional on `clinical_validation_status`, so unverified documents are publicly searchable — compensated by keeping `unverified_source` in the frontend-visible warning set. A hard schema flag touches RLS and the clinical-risk-gated retrieval RPCs; weigh against the existing compensating control before acting. | `supabase/schema.sql:61-108`; `src/lib/search-scope.ts:181-236`; PR #1051 audit item 3 | 2026-07-22 | +| #037 | P3 | rec | D5 trust-cap-all-claims flag parked OFF | `NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS` extends authority gating from high-risk claims to **all** supported claims (`deriveTrust`). Ships OFF by design; flipping it caps trust to `medium` for routine claims across the board — a product/clinical-UX decision, not a defect. Both states are test-pinned. Next action: product decision, then flip and re-baseline the UI expectations. | `src/lib/answer-render-policy.ts:159-177`; PR #1051 audit item 11 | 2026-07-22 | +| #038 | P3 | rec | Consolidate shared comparison behavior | Several clinical modes expose comparison workflows with similar selection, empty-state and mobile-dock needs. Define one shared behavioral contract before another comparison surface is added; keep mode-specific clinical content separate. This is a design-system recommendation, not a current defect. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #039 | P3 | rec | Consolidate catalogue toolbar patterns | Catalogue/search pages have independently evolved filter, sort, result-count and mobile toolbar behavior. Inventory the existing implementations and converge only the repeated interaction contract; do not flatten mode-specific search semantics. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #041 | P3 | rec | Extend the existing Factsheets reading model | Do not add a second patient-facing Factsheets mode. Future patient-content work should extend the existing Easy Read/Standard presentation and its accessibility/content contracts. Revisit only with a concrete user need and source-governance plan. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | ## Resolved / archive From 40010068ff38edbbfa46ead68c29e598ef24c9a8 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:25:54 +0800 Subject: [PATCH 4/4] docs: distinguish identical-tree canary runs Use the green main baseline and reverted confirmation as the byte-identical comparison, and describe the candidate run separately. --- docs/outstanding-issues.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index ad0fac530..1c2792270 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -35,7 +35,7 @@ Durable, cross-session memory of everything still outstanding for this repo: ope | ID | Pri | Type | Summary | Detail / next action | Source | Added | | ---- | --- | ----- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| #051 | P2 | task | Stabilise the live answer-quality canary before more RAG tuning | Two full canaries on different commits with byte-identical trees passed golden retrieval (document/content recall 1.0 and zero per-case reciprocal-rank regressions) but failed the answer gate for different reasons: candidate run `30007833352` exceeded unsupported-route p95 (6,227 ms > 4,000 ms), while reverted confirmation `30009207429` had citation-failure rate 0.0227 from the neuroleptic case selecting one wrong source. Do not spend on another immediate retry. First make the answer gate distinguish provider/latency variability from deterministic content regressions and add stable per-case diagnostics; then re-evaluate the archived lithium guard. The rejected commit is recoverable at local ref `refs/archive/rejected-rag/20260723/monitoring-subject-gate`. | Eval Canary runs `30007833352`, `30009207429`; PR #1093; session 2026-07-23 | 2026-07-23 | +| #051 | P2 | task | Stabilise the live answer-quality canary before more RAG tuning | Baseline main run `30005641859` and reverted confirmation `30009207429` used byte-identical trees: the baseline passed, while confirmation failed on citation-failure rate 0.0227 from the neuroleptic case, proving that the live answer gate can vary without a tree change. The separate candidate run `30007833352` kept golden recall 1.0 with zero per-case reciprocal-rank regressions but exceeded unsupported-route p95 (6,227 ms > 4,000 ms). Do not spend on another immediate retry. First make the answer gate distinguish provider/latency variability from deterministic content regressions and add stable per-case diagnostics; then re-evaluate the archived lithium guard. The rejected commit is recoverable at local ref `refs/archive/rejected-rag/20260723/monitoring-subject-gate`. | Eval Canary runs `30007833352`, `30009207429`; PR #1093; session 2026-07-23 | 2026-07-23 | | #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | | #005 | P3 | rec | `finalScore` saturates at clamp ceiling | Base + ~40 stacked boosts routinely exceed 1.0, so strong matches tie at 1.0 and order by an arbitrary `document_id` tiebreak. If ranking is ever revisited, break ties by the **pre-clamp** score rather than raising the `[0,1]` ceiling (downstream gates assume `[0,1]`). Ordering already sorts by the unbounded pre-clamp `rankScore` (`clinical-search.ts:1735,1927,1950-1955`), so the clamp confines only the reported confidence value, not result order. Not a defect on the current golden set; any change here is a protected RAG surface (canary required). | `docs/rag-hybrid-findings-and-todo.md` P1 item 4; `src/lib/clinical-search.ts:1735` | 2026-07-21 | | #007 | P3 | rec | `/tools` vs `/?mode=tools` parallel Tools entry points | `/tools` (standalone `ApplicationsLauncherPage`) has no inbound in-app link; the sidebar Tools item uses `/?mode=tools`. Decide the canonical entry point and wire nav consistently, or drop the standalone `/tools` page + `/applications` redirect. Currently allowlisted in `tests/route-reachability.test.ts`. | `src/app/tools/page.tsx`; `src/app/applications/route.ts` | 2026-07-21 |