feat(eval): canary retrieval artifact + alias-aware ranking-snapshot builder with freshness provenance (ADDENDUM 4 A-PR-1)

.github/workflows/eval-canary.yml

@@ -3,7 +3,8 @@
 # Runs the golden retrieval eval (which PR CI can never run — it needs live
 # Supabase + OpenAI keys) plus a small answer-quality subset against the live
 # project, and fails loudly on regression: red run + a GitHub issue on
-# scheduled failures.
+# scheduled failures. Each run also uploads the eval JSON artifact that feeds
+# offline ranking-snapshot regeneration (scripts/build-ranking-snapshot.ts).
 #
 # The schedule only fires from the default branch. After merging, trigger one
 # workflow_dispatch run and confirm it is green before trusting the weekly
@@ -97,7 +98,10 @@ jobs:
         run: |
           mkdir -p .local/eval-canary
           set -o pipefail
-          npm run eval:retrieval:quality -- --fail-on-threshold 2>&1 | tee .local/eval-canary/golden-retrieval.log
+          # --json-out writes the per-case artifact (input for scripts/build-ranking-snapshot.ts)
+          # without changing stdout — the tee'd log keeps the human-readable per-case lines the
+          # failure-issue analyzer parses.
+          npm run eval:retrieval:quality -- --fail-on-threshold --json-out .local/eval-canary/golden-retrieval.json 2>&1 | tee .local/eval-canary/golden-retrieval.log
 
       - name: Answer-quality subset (live generation)
         id: answer_quality
@@ -115,6 +119,25 @@ jobs:
           set -o pipefail
           npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
 
+      # Uploaded on success AND failure: the JSON artifact regenerates the offline ranking
+      # snapshot (download it, then `npm run build:ranking-snapshot -- --input <json> --output
+      # scripts/fixtures/rag-ranking-candidate-snapshot.v1.json --source-run-id <run-id>`), and
+      # the logs carry failure diagnostics. Contents match what the tee'd step logs already
+      # expose (titles/file names, score telemetry, 220-char previews of the all-public corpus)
+      # — no env values or secrets are written to .local/eval-canary.
+      - name: Upload eval output artifact
+        if: always() && steps.install.outcome == 'success'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: eval-canary-output
+          path: .local/eval-canary/
+          # .local/ is a hidden directory (the repo's gitignored scratch convention), which
+          # upload-artifact excludes by default.
+          include-hidden-files: true
+          # Matches the snapshot freshness window enforced by tests/ranking-tuning.test.ts.
+          retention-days: 30
+          if-no-files-found: ignore
+
       - name: Open or update canary failure issue
         if: failure() && github.event_name == 'schedule'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

docs/branch-review-ledger.md

@@ -650,3 +650,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD
 | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: coverage tie-break follow-up) | 57ec880b306a8e2b31c5f20dacc47256fc93b4e2 | Post-merge live-eval finding on #982 + corrective fix: saturated-tie key rankScore → query-term coverage | Post-#982 golden dispatch (eval-canary run #50, 29735004222, main b9057f0 + deps) came back 35/36: the three verifiable July-19 failures (lithium-therapy-monitoring, clozapine-anc-threshold, patient-safety-plan-include) all PASS live, but alcohol-ciwa-threshold flipped pass→FAIL vs the same-morning pre-#982 run #49 (29731533081, 36/36) — failing top-3 ordered by descending rankScore (1.85/1.75/1.53, all finalScore-saturated, releaseRankScore 1.09/1.086/1.07), i.e. #982's tie-break let generic clinicalSignalBoost stacking outvote the ciwa/score/threshold-bearing chunk; #982 is the only retrieval-path delta in the window. Fix: contentRankScore → contentCoverageScore sourced from lexicalCoverageScore (query-term coverage, immune to boost stacking; ties still fall to chunk id); saturated-tie contract test re-pinned so coverage beats a HIGHER rankScore (discriminating — old key fails it); fast-path CIWA guard gains the run-#50 screening-chunk shape + content-term assertion. Live validation: tonight's 18:00 UTC scheduled canary (dispatch cap 2/2 spent ≈$2-4). Separately: ci.yml dispatch 4012 survived 30+ min of main churn under #979's per-run concurrency group (fix working); duplicate dispatch 4017 cancelled. | Targeted vitest 38/38; npm run test 3012 passed / 1 known container-only pdf-budget artifact; verify:cheap green to the same artifact; build + client-bundle scan + check:rag:fixtures PASS; check:production-readiness expected missing-secret FAILs only (no secrets in container) |
 | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: WebKit matrix diagnosis) | caa298972846ef52d472bfbff98c90fc26481ed8 | First completed release-browser-matrix in weeks (run 4012, main b9057f0, 38 min uncancelled under #979) — full triage of 28 failures + fix for the dominant cluster | Run 4012: 716 passed / 28 failed (25 webkit, 3 firefox) / 4 skipped. Dominant cluster (root-caused, FIXED here): all 8 ui-universal-search webkit failures share one signature — typeahead content never enters the DOM — because commandDropdownCanDisplay (added 42a3e3c 2026-07-17, AFTER the last completed matrix; never ran on WebKit until 4012) requires fine-pointer OR zero-touch; headless browsers fail the (hover:hover)+(pointer:fine) query (proven by the zero-touch escape existing for CI at all), Chromium/Firefox pass via maxTouchPoints===0, and Playwright's Linux WebKit build advertises phantom touch points — flunking both branches and disabling useUniversalSearch entirely. Fix: beforeEach addInitScript in ui-universal-search/ui-smoke/ui-tools specs stubbing Navigator.prototype.maxTouchPoints to the runner's true 0 (inert on Chromium/Firefox; product gate + tests/search-command-surface.test.ts pins untouched). Local chromium runs of ui-universal-search fail 16/20 IDENTICALLY on unmodified main (container artifact — hosted CI chromium green in 4012 is authoritative; verified by stash/run/pop baseline). Expected delta next matrix: ≥8 webkit failures clear; candidates ui-tools:1244 + several ui-smoke answer-flow cases (same surface). NOT yet root-caused (triaged remainder, hosted-matrix-only reproduction): webkit ui-stress overflow 330/409 (360px overflow at mobile), webkit ui-accessibility 195/252 (focus dismissal; forced-colors labels), webkit ui-formulation 132, webkit ui-smoke copy-table/retry/recovery/recent-searches/source-only/differential-context/viewer-hydration/document-questions, webkit ui-tools 2087 (goto interrupted by ?q= navigation), webkit ui-universal-search 183 strict-mode duplicate options (fallback-surface rendering; likely clears with the gate fix), firefox ui-smoke 946/2206/2932 (@critical document search fails on BOTH firefox and webkit — cross-browser, highest-priority remainder). | prettier PASS; eslint PASS; typecheck PASS; chromium stash-baseline no-delta (16 fail pre AND post — container artifact); webkit validation lands via tonight's 18:00 UTC scheduled matrix |
 | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: matrix-remainder fixes) | ec070ccb20b9a067216e64e8399d9cd795027024 | Run-4012 remainder: 7 root-caused fixes (SW navigation hijack, WebKit focus/forced-colors/stale-style, Firefox tab order, CIWA alias, helper extraction) | Agent-diagnosed with probe evidence: (1) production-build PWA worker registers in EVERY matrix test (pwa-lifecycle.tsx:287 NODE_ENV gate), clients.claim()s the page and serves all navigations — Chromium probe proved SW-served reloads bypass page.route entirely (routeSawNav=[]); Playwright-Firefox wedges on its only two reloads (ui-smoke 2206/2932, both deterministic) → playwright.config.ts serviceWorkers:'block' + ui-pwa 'allow' opt-in (offline/CacheStorage journey verified passing locally under the opt-in). (2) Mode-menu dismissal relied solely on wrapper focusout; WebKit Tab navigation can move focus nowhere (links excluded) or wrap into the menu → keydown Tab-close in handleModeTriggerKeyDown (agent verified no test depends on old forward-Tab behavior). (3) WebKit has no forced-colors implementation → capability skip on the token-remap test. (4) WebKit stale :disabled computed style feeds axe a phantom 1.93 contrast for the re-enabled Previous button (blend arithmetic exact: 0.4×#475467+0.6×#fff=#b5bbc2) → toBeEnabled+opacity-1 pin before the scan, failure-at-pin = direct proof. (5) Firefox includes scrollable containers in tab order (sheet body scrollHeight 1125 vs 707 measured) → conditional step-over. (6) canary #50/#51 CIWA failure root cause: whitespace-delimited textContainsClinicalTerm can never match hyphenated 'CIWA-Ar' though the dosing-table region ranks top-5 → clinicalContentAliases ciwa:[ciwa, ciwa-ar] (plan-authorized fixture-alias route; content substance unchanged). (7) tests/helpers/zero-touch.ts shared helper replaces six #995 inline stubs (CodeRabbit follow-up). NOT claimed: ui-stress 409 overflow, ui-tools 2087 navigation race, webkit answer-flow subset — next matrix run (post-merge dispatch) measures these vs the 28-failure baseline; ui-smoke 2932 webkit-side is a distinct mock-data class deliberately deferred pending that run. | Targeted vitest 25/25 (eval-retrieval + search-command-surface); npm run test 3012 passed / 1 known container pdf-budget artifact; verify:cheap green to same artifact; build + client-bundle scan PASS; prettier/eslint/typecheck clean; chromium ui-pwa under new SW config: offline/CacheStorage privacy journey PASS, installability = known container in-incognito artifact; UI verification not run locally beyond that: chromium ui suites carry documented container artifacts — hosted ui-critical/ui-advisory + next matrix authoritative |
+| 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted; PR: eval measurement floor, ADDENDUM 4 A-PR-1) | 81ab9696da4b330ca0b2e5519891a9942f90421b | Measurement floor for evidence-gated ranking tuning: canary artifact emission, alias-aware snapshot builder, snapshot provenance/freshness — no ranking behavior change | Closes the three gaps blocking safe tuning (Phase B): (1) eval-canary's golden step now writes the per-case JSON artifact (--json-out decoupled from --json so the tee'd log keeps the human-readable lines the failure-issue analyzer parses) and uploads .local/eval-canary/ via pinned upload-artifact (30-day retention, include-hidden-files for the dot-dir, contents = same class as the already-public step logs: titles/telemetry/220-char previews of the all-public corpus) — snapshot regeneration stops costing a paid dispatch; (2) clinicalDocumentAliases/clinicalContentAliases moved verbatim to shared scripts/lib/clinical-aliases.ts and the snapshot builder grades documentMatch/contentMatch through them (discriminating tests: EMHS agitation title and spelled-out "absolute neutrophil count" grade as hits only via aliases — raw labelMatches pinned false), ending tuner ground truth disagreeing with the live gates; (3) snapshots carry generatedAt + optional sourceRunId, validator accepts them, exactly-36 relaxed to at-least-36 (floor still rejects truncated artifacts; sourceCaseCount + per-case candidate minimums unchanged), and a 30-day freshness test (activates on first regeneration) blocks silent corpus drift. Builder smoke-verified end-to-end on a synthetic 36-case artifact (alias grading + provenance stamped + validator green). Static hyphen audit of all 36 cases' terms: no currently-blocked term (canary #52 = 36/36); residual risk classes documented for the A-PR-2 artifact-grounded pass — punctuation-joined tokens (IM/PO, schizo-affective, post-natal) and inert stem entries (obsess/compuls/hyperactiv/impuls can never match whole-token) that currently ride on whole-word OR-alternates. | Targeted vitest 52/52 (ranking-tuning + eval-retrieval + eval-quality); npm run test 3019 passed / 1 known container-only pdf-budget artifact; lint + typecheck clean; check:github-actions + check:ci-scope PASS; prettier clean; check:production-readiness expected missing-secret FAILs only (demo-mode container); no provider calls — live validation = tonight's scheduled canary emits the first artifact at $0 |

package.json

@@ -151,6 +151,7 @@
     "profile:retrieval": "node scripts/run-tsx.mjs scripts/profile-retrieval-rpcs.ts",
     "warm:retrieval": "node scripts/run-tsx.mjs scripts/warm-retrieval-cache.ts",
     "tune:search-weights": "node scripts/run-tsx.mjs scripts/tune-search-weights.ts",
+    "build:ranking-snapshot": "node scripts/run-tsx.mjs scripts/build-ranking-snapshot.ts",
     "cleanup:storage": "node scripts/run-tsx.mjs scripts/cleanup-storage.ts",
     "purge:query-logs": "node scripts/run-tsx.mjs scripts/purge-query-logs.ts",
     "audit:tables": "node scripts/run-tsx.mjs scripts/audit-tables.ts",

scripts/build-ranking-snapshot.ts

@@ -2,7 +2,12 @@ import { createHash } from "node:crypto";
 import { readFileSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
 import type { RagQueryClass } from "../src/lib/types";
-import { candidateFeatures, labelMatches, type ArtifactCandidate } from "./lib/ranking-snapshot-builder";
+import {
+  candidateFeatures,
+  contentLabelMatchesWithAliases,
+  documentLabelMatchesWithAliases,
+  type ArtifactCandidate,
+} from "./lib/ranking-snapshot-builder";
 import {
   RANKING_SNAPSHOT_VERSION,
   type RankingCandidateFeatures,
@@ -73,19 +78,26 @@ const hardNegativeTemplates: Array<{
   features: RankingCandidateFeatures;
 }>;
 
-function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot {
-  if (!Array.isArray(artifact.results) || artifact.results.length !== 36) {
-    throw new Error("Expected a retrieval artifact containing exactly 36 cases");
+function convertArtifact(artifact: RetrievalArtifact, sourceRunId?: string): RankingSnapshot {
+  // Floor, not an exact pin: the golden fixture only ever grows, and a short artifact means a
+  // truncated or filtered eval run that must not silently become the tuner's ground truth.
+  if (!Array.isArray(artifact.results) || artifact.results.length < 36) {
+    throw new Error(
+      `Expected a retrieval artifact containing at least 36 cases (got ${
+        Array.isArray(artifact.results) ? artifact.results.length : 0
+      })`,
+    );
   }
   const cases = artifact.results.map((testCase) => {
     const expectedDocuments = testCase.expectedDocumentSubstrings ?? [];
     const expectedContent = testCase.expectedContentTerms ?? [];
     const candidates = (testCase.topResults ?? []).map((candidate, index): RankingSnapshotCandidate => {
       const documentText = `${candidate.title ?? ""} ${candidate.file_name ?? ""}`.toLowerCase();
       const contentText = (candidate.content_preview ?? "").toLowerCase();
-      const documentMatch = expectedDocuments.some((label) => labelMatches(documentText, label));
+      const documentMatch = expectedDocuments.some((label) => documentLabelMatchesWithAliases(documentText, label));
       const contentMatch =
-        expectedContent.length > 0 && expectedContent.every((label) => labelMatches(contentText, label));
+        expectedContent.length > 0 &&
+        expectedContent.every((label) => contentLabelMatchesWithAliases(contentText, label));
       return {
         candidateHash: candidateHash(`${testCase.id}:${candidate.chunk_id ?? `rank-${index + 1}`}`),
         relevanceGrade: documentMatch && contentMatch ? 3 : documentMatch ? 2 : contentMatch ? 1 : 0,
@@ -116,6 +128,10 @@ function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot {
     schema: "rag-ranking-candidate-snapshot",
     version: RANKING_SNAPSHOT_VERSION,
     sourceCaseCount: cases.length,
+    // Freshness provenance: tests/ranking-tuning.test.ts fails the build when generatedAt is
+    // older than 30 days, so a checked-in snapshot cannot silently drift from the live corpus.
+    generatedAt: new Date().toISOString(),
+    ...(sourceRunId ? { sourceRunId } : {}),
     sanitization: {
       candidateIdentity: "sha256",
       excludes: ["raw_uuid", "source_passage", "patient_data", "provider_metadata", "document_storage_path"],
@@ -127,10 +143,15 @@ function convertArtifact(artifact: RetrievalArtifact): RankingSnapshot {
 function main() {
   const input = argument("--input");
   const output = argument("--output");
+  // Optional provenance: the GitHub Actions run id of the eval-canary run whose artifact
+  // produced --input, so a snapshot can be traced back to its live eval.
+  const sourceRunId = argument("--source-run-id");
   if (!input || !output)
-    throw new Error("Usage: build-ranking-snapshot --input <artifact.json> --output <snapshot.json>");
+    throw new Error(
+      "Usage: build-ranking-snapshot --input <artifact.json> --output <snapshot.json> [--source-run-id <actions-run-id>]",
+    );
   const artifact = JSON.parse(readFileSync(resolve(input), "utf8")) as RetrievalArtifact;
-  const snapshot = validateRankingSnapshot(convertArtifact(artifact));
+  const snapshot = validateRankingSnapshot(convertArtifact(artifact, sourceRunId));
   writeFileSync(resolve(output), `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
   console.log(JSON.stringify({ output: resolve(output), cases: snapshot.cases.length, version: snapshot.version }));
 }