diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index f609081e5..0acd9bb2c 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -81,7 +81,7 @@ removed after current-main verification; it is not missing recommended work. | 29 | `#065` | A2 | High — document-viewer UI | Only when the user explicitly resumes the paused task | 0.5–1.5 days | Finish the compact source-text accordion, citation/search auto-open, print restoration, and 320/390/1280 px coverage. Keep the preserved branch untouched until explicit resume; no provider calls. | | 30 | `#079` | Optional | High — repository hygiene | In explicitly scheduled batches | 30–60 minutes per batch | Disposition at most ten retained worktrees per pass using owner, PR, review-ledger, ancestry, and patch evidence. Preserve every dirty, active, secret-bearing, post-freeze, or ambiguous worktree and stop rather than broad-cleaning. | - + ## Open items @@ -130,6 +130,7 @@ removed after current-main verification; it is not missing recommended work. | #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | | #079 | P3 | task | Disposition retained worktrees in bounded cleanup batches | **Outcome:** the retained reconciliation tail is gradually classified without another disruptive all-worktree sweep. **Next:** process no more than ten worktrees per explicitly scheduled pass using current owner/process metadata, open-PR state, exact review-ledger coverage, ancestry, and cherry-pick-aware content proof. **Success:** remove only clean, inactive, bundled worktrees whose content is merged or explicitly rejected; record every disposition and retain recovery evidence. **Stop:** preserve dirty, active, secret-bearing, post-freeze, paused, or ambiguous work and never use reset, force deletion, broad clean, or process killing. | final reconciliation inventory retained 104 independent worktrees; session 2026-07-24 | 2026-07-24 | | #080 | P3 | rec | Re-test the removed admission-to-discharge alias widening | **Outcome:** decide with measured evidence whether the user-approved 2026-07-21 widening (NMHS admission-to-discharge titles satisfying the admission expectation) belongs back on the wide-tier `AdmissionCommunityPts` list. #030 removed it so one document could not fill both comparison slots, but coverage now enforces distinct document identities by maximum matching, so the widening is no longer a false-pass route and its removal may instead cost legitimate admission credit. **Next:** with approval, run an eval-canary baseline/post pair over the admission-discharge comparison cases with and without those two titles. **Success:** restore only on measured non-regression (recall pinned 1.0, zero per-case reciprocal-rank regressions); otherwise record keep-removed and close. **Stop:** never restore on offline reasoning alone. | `src/lib/eval-document-matching.ts`; #030 archive row; session 2026-07-25 | 2026-07-25 | +| #081 | P2 | issue | Open PR #1196 would undo the #030 alias tightening | **Outcome:** #1196 (`codex/fix-p2-audit-20260719`, 74 files, based on `e4d88b47`) reconciles with main without reverting #030. It re-adds "Admission to Discharge for Community Mental Health" and "Referral Admission and Discharge Mental Health Hospital in the Home" to wide-tier `AdmissionCommunityPts`, where both already sit on the Discharge tier. **Next:** when that branch is synced, resolve `src/lib/eval-document-matching.ts` by taking main's admission-only list. Its other conflicts include protected RAG surfaces (`semantic-rerank.ts`, `rag/rag.ts`, `rag-route-budget.ts`, `scripts/eval-retrieval.ts`), so it needs the RAG flagging path, not a mechanical rebase. **Success:** `tests/eval-document-matching.test.ts` stays green on the reconciled head. **Stop:** never take the #1196 side of the alias conflict. | PR #1196 conflict inventory; contract fails on its alias change; session 2026-07-25 | 2026-07-25 | ## Resolved / archive diff --git a/tests/eval-document-matching.test.ts b/tests/eval-document-matching.test.ts index 0f7e8a96a..ee0f7f6e5 100644 --- a/tests/eval-document-matching.test.ts +++ b/tests/eval-document-matching.test.ts @@ -4,6 +4,17 @@ import { expectedFileCoverage, normalizedDocumentName, } from "@/lib/eval-document-matching"; +import { answerQualityEvalCases, ragEvalCases } from "@/lib/rag/rag-eval-cases"; + +const multiSlotCases = [...ragEvalCases, ...answerQualityEvalCases].filter( + (evalCase) => evalCase.expectedFiles.length > 1, +); + +// Alias values a wide-tier expectation recognizes, excluding the expectation's own name. +function aliasValuesFor(expectation: string) { + const own = normalizedDocumentName(expectation); + return documentExpectationAlternatives(expectation).filter((value) => value !== own); +} describe("eval document matching wide-tier aliases", () => { it("does not let one dual-listed admission-to-discharge doc satisfy both comparison slots", () => { @@ -104,4 +115,48 @@ describe("eval document matching wide-tier aliases", () => { expect(expectedFileCoverage(["CG.MHSP.Lithium.pdf", "CG.MHSP.Metformin.pdf"], sources, 5).allHit).toBe(true); expect(expectedFileCoverage(["CG.MHSP.Metformin.pdf", "CG.MHSP.Lithium.pdf"], sources, 5).allHit).toBe(true); }); + + it("keeps the alias values of every multi-slot case's expectations pairwise disjoint", () => { + // Generalizes the admission/discharge check above to every multi-slot case, including any + // added later. A value listed under two slots of the same case is the #030 defect: it lets + // one document stand in for two independent sources. + const overlaps = multiSlotCases.flatMap((evalCase) => { + const found: string[] = []; + for (let i = 0; i < evalCase.expectedFiles.length; i += 1) { + for (let j = i + 1; j < evalCase.expectedFiles.length; j += 1) { + const left = new Set(aliasValuesFor(evalCase.expectedFiles[i])); + const shared = aliasValuesFor(evalCase.expectedFiles[j]).filter((value) => left.has(value)); + for (const value of shared) { + found.push( + `${evalCase.id}: "${value}" is listed under both ${evalCase.expectedFiles[i]} and ${evalCase.expectedFiles[j]}`, + ); + } + } + } + return found; + }); + + expect(overlaps).toEqual([]); + }); + + it("never lets a single document satisfy every slot of a multi-slot case", () => { + // Table-independent: whatever the alias tables say, one retrieved document is one source. + // This holds even for a document whose text contains every expectation, so it stays true if + // the alias tables are widened again. + expect(multiSlotCases.length).toBeGreaterThan(0); + + for (const evalCase of multiSlotCases) { + const everyExpectation = evalCase.expectedFiles + .flatMap((expectation) => [expectation, ...aliasValuesFor(expectation)]) + .join(" "); + const coverage = expectedFileCoverage( + evalCase.expectedFiles, + [{ title: everyExpectation, file_name: `${everyExpectation}.pdf` }], + 5, + ); + + expect(coverage.matchedFiles.length, `${evalCase.id} matched more than one slot from one document`).toBe(1); + expect(coverage.allHit, `${evalCase.id} reached allHit from a single document`).toBe(false); + } + }); });