From 8dedbfa0601ddd12f1c93fb92dfa48f4485b1953 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:24:54 +0800 Subject: [PATCH 1/4] fix(rag): restore live-validated retrieval order --- src/lib/rag.ts | 25 ++++++++++++-- tests/rag-second-stage-ranking.test.ts | 45 +++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 82d424ca5..61ef0efba 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -549,6 +549,28 @@ function recordRetrievalLayer( } } +/** + * Keep the released result order on the live-eval-proven hybrid signal. + * + * App-layer rank scores remain available to answer evidence ranking and telemetry, but the + * live corpus gate has not validated using them as the final retrieval order. Sort before + * deduplication so the strongest copy of a duplicated chunk remains the one returned. + */ +export function stabilizeReleasedSearchOrder(results: SearchResult[]) { + results.sort((left, right) => { + const leftHybrid = left.hybrid_score ?? left.similarity ?? 0; + const rightHybrid = right.hybrid_score ?? right.similarity ?? 0; + if (rightHybrid !== leftHybrid) return rightHybrid - leftHybrid; + const leftSimilarity = left.similarity ?? 0; + const rightSimilarity = right.similarity ?? 0; + if (rightSimilarity !== leftSimilarity) return rightSimilarity - leftSimilarity; + if (right.relevance?.score !== left.relevance?.score) + return (right.relevance?.score ?? 0) - (left.relevance?.score ?? 0); + return left.id.localeCompare(right.id); + }); + return results; +} + /** Record search score telemetry. */ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) { if (!results.length) { @@ -565,8 +587,7 @@ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchR return; } - // Preserve the rankScore ordering established by clinical/second-stage ranking. Coverage - // telemetry still uses the raw hybrid signal, but must never mutate the returned result order. + stabilizeReleasedSearchOrder(results); const deduped: SearchResult[] = []; const seen = new Set(); for (const result of results) { diff --git a/tests/rag-second-stage-ranking.test.ts b/tests/rag-second-stage-ranking.test.ts index 70d28484e..d75c92924 100644 --- a/tests/rag-second-stage-ranking.test.ts +++ b/tests/rag-second-stage-ranking.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { applySecondStageRerankIfNeeded } from "../src/lib/rag"; +import { applySecondStageRerankIfNeeded, stabilizeReleasedSearchOrder } from "../src/lib/rag"; import type { SearchTelemetry } from "../src/lib/rag-contracts"; import type { SearchResult, SearchScoreExplanation } from "../src/lib/types"; @@ -136,4 +136,47 @@ describe("second-stage rank score", () => { expect(ranked.map((item) => item.id)).toEqual(["dose-evidence-without-explanation", "supported"]); expect(ranked[1].score_explanation?.finalRank).toBe(2); }); + + it("keeps unvalidated rank-score changes from replacing the live-eval-proven release order", () => { + const telemetry = {} as SearchTelemetry; + const strongerHybrid = result({ + id: "stronger-hybrid", + document_id: "older-doc", + hybrid_score: 0.8, + score_explanation: explanation(0.4), + source_metadata: { + source_title: "Older guideline", + publisher: "Test publisher", + jurisdiction: "Australia/WA", + version: "1", + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "outdated", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }, + }); + const higherRankScore = result({ + id: "higher-rank-score", + document_id: "current-doc", + hybrid_score: 0.79, + score_explanation: explanation(0.5), + }); + + const secondStage = applySecondStageRerankIfNeeded({ + queryClass: "medication_dose_risk", + results: [strongerHybrid, higherRankScore], + telemetry, + topK: 2, + }); + expect(secondStage.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid"]); + + expect(stabilizeReleasedSearchOrder(secondStage).map((item) => item.id)).toEqual([ + "stronger-hybrid", + "higher-rank-score", + ]); + }); }); From a18b08e487ca6d234843f5388fe26d120647c31a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:34:44 +0800 Subject: [PATCH 2/4] fix(rag): preserve answer ranking tie-breaks --- src/lib/answer-ranking.ts | 1 + tests/answer-ranking.test.ts | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts index f4dbcb1ef..406ff2a9e 100644 --- a/src/lib/answer-ranking.ts +++ b/src/lib/answer-ranking.ts @@ -249,6 +249,7 @@ export function rankAnswerEvidence( (a, b) => b.score - a.score || (b.result.hybrid_score ?? b.result.similarity ?? 0) - (a.result.hybrid_score ?? a.result.similarity ?? 0) || + (b.result.score_explanation?.rankScore ?? 0) - (a.result.score_explanation?.rankScore ?? 0) || a.index - b.index, ); diff --git a/tests/answer-ranking.test.ts b/tests/answer-ranking.test.ts index fd0a591a9..b7b0d2668 100644 --- a/tests/answer-ranking.test.ts +++ b/tests/answer-ranking.test.ts @@ -179,6 +179,26 @@ describe("answer evidence ranking", () => { ranking.scoresByChunkId.get("low-semantic") ?? 0, ); }); + + it("uses the app-layer rank only as a final answer-evidence tie-breaker", () => { + const lowerRank = result({ + id: "lower-rank", + content: "Lithium dosing guidance.", + hybrid_score: 0.7, + score_explanation: { rankScore: 0.8 } as NonNullable, + }); + const higherRank = result({ + id: "higher-rank", + content: "Lithium dosing guidance.", + hybrid_score: 0.7, + score_explanation: { rankScore: 1.1 } as NonNullable, + }); + + const ranking = rankAnswerEvidence("lithium dosing guidance", [lowerRank, higherRank]); + + expect(ranking.rankedResults.map((item) => item.id)).toEqual(["higher-rank", "lower-rank"]); + expect(ranking.scoresByChunkId.get("higher-rank")).toBe(ranking.scoresByChunkId.get("lower-rank")); + }); }); describe("high-yield answer bolding", () => { From 721c1be2f54b44c880a415d8474233a9c2d0b970 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 07:41:04 +0000 Subject: [PATCH 3/4] fix(rag): preserve ranked fallback order while deduping duplicates --- src/lib/rag.ts | 33 +++++++++++++++----------- tests/rag-second-stage-ranking.test.ts | 24 +++++++++++++------ 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 1d18e682b..5c9e068c4 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -550,14 +550,14 @@ function recordRetrievalLayer( } /** - * Keep the released result order on the live-eval-proven hybrid signal. + * Keep distinct results in the current ranked order while upgrading duplicate chunks to the + * strongest released-hybrid copy. * * App-layer rank scores remain available to answer evidence ranking and telemetry, but the - * live corpus gate has not validated using them as the final retrieval order. Sort before - * deduplication so the strongest copy of a duplicated chunk remains the one returned. + * live corpus gate has not validated replacing the final distinct-result order with them. */ export function stabilizeReleasedSearchOrder(results: SearchResult[]) { - results.sort((left, right) => { + const compareReleasedSearchStrength = (left: SearchResult, right: SearchResult) => { const leftHybrid = left.hybrid_score ?? left.similarity ?? 0; const rightHybrid = right.hybrid_score ?? right.similarity ?? 0; if (rightHybrid !== leftHybrid) return rightHybrid - leftHybrid; @@ -567,7 +567,21 @@ export function stabilizeReleasedSearchOrder(results: SearchResult[]) { if (right.relevance?.score !== left.relevance?.score) return (right.relevance?.score ?? 0) - (left.relevance?.score ?? 0); return left.id.localeCompare(right.id); - }); + }; + const strongestById = new Map(); + for (const result of results) { + const current = strongestById.get(result.id); + if (!current || compareReleasedSearchStrength(result, current) < 0) strongestById.set(result.id, result); + } + const deduped: SearchResult[] = []; + const seen = new Set(); + for (const result of results) { + if (seen.has(result.id)) continue; + seen.add(result.id); + deduped.push(strongestById.get(result.id) ?? result); + } + results.length = 0; + results.push(...deduped); return results; } @@ -588,15 +602,6 @@ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchR } stabilizeReleasedSearchOrder(results); - const deduped: SearchResult[] = []; - const seen = new Set(); - for (const result of results) { - if (seen.has(result.id)) continue; - seen.add(result.id); - deduped.push(result); - } - results.length = 0; - results.push(...deduped); const coverageScores = results .map((result) => Math.max(0, result.hybrid_score ?? result.similarity ?? 0)) .sort((left, right) => right - left); diff --git a/tests/rag-second-stage-ranking.test.ts b/tests/rag-second-stage-ranking.test.ts index d75c92924..caf18b936 100644 --- a/tests/rag-second-stage-ranking.test.ts +++ b/tests/rag-second-stage-ranking.test.ts @@ -137,9 +137,9 @@ describe("second-stage rank score", () => { expect(ranked[1].score_explanation?.finalRank).toBe(2); }); - it("keeps unvalidated rank-score changes from replacing the live-eval-proven release order", () => { + it("keeps distinct rank order while upgrading duplicate chunks to the strongest released-hybrid copy", () => { const telemetry = {} as SearchTelemetry; - const strongerHybrid = result({ + const strongerHybridDuplicate = result({ id: "stronger-hybrid", document_id: "older-doc", hybrid_score: 0.8, @@ -165,18 +165,28 @@ describe("second-stage rank score", () => { hybrid_score: 0.79, score_explanation: explanation(0.5), }); + const weakerDuplicate = { + ...strongerHybridDuplicate, + hybrid_score: 0.78, + score_explanation: explanation(0.41), + content: "Weaker duplicate", + }; const secondStage = applySecondStageRerankIfNeeded({ queryClass: "medication_dose_risk", - results: [strongerHybrid, higherRankScore], + results: [higherRankScore, weakerDuplicate, strongerHybridDuplicate], telemetry, - topK: 2, + topK: 3, }); - expect(secondStage.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid"]); + expect(secondStage.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid", "stronger-hybrid"]); - expect(stabilizeReleasedSearchOrder(secondStage).map((item) => item.id)).toEqual([ - "stronger-hybrid", + const stabilized = stabilizeReleasedSearchOrder(secondStage); + + expect(stabilized.map((item) => item.id)).toEqual([ "higher-rank-score", + "stronger-hybrid", ]); + expect(stabilized[1].hybrid_score).toBe(0.8); + expect(stabilized[1].content).toBe(strongerHybridDuplicate.content); }); }); From c4eb3ef9fb7f40a7b2482e14cb9831cbd85ca9d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 07:41:30 +0000 Subject: [PATCH 4/4] fix(rag): preserve ranked fallback order while deduping duplicates --- tests/rag-second-stage-ranking.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/rag-second-stage-ranking.test.ts b/tests/rag-second-stage-ranking.test.ts index caf18b936..52c7dfe81 100644 --- a/tests/rag-second-stage-ranking.test.ts +++ b/tests/rag-second-stage-ranking.test.ts @@ -182,10 +182,7 @@ describe("second-stage rank score", () => { const stabilized = stabilizeReleasedSearchOrder(secondStage); - expect(stabilized.map((item) => item.id)).toEqual([ - "higher-rank-score", - "stronger-hybrid", - ]); + expect(stabilized.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid"]); expect(stabilized[1].hybrid_score).toBe(0.8); expect(stabilized[1].content).toBe(strongerHybridDuplicate.content); });