diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts index f4dbcb1ef..406ff2a9e 100644 --- a/src/lib/answer-ranking.ts +++ b/src/lib/answer-ranking.ts @@ -249,6 +249,7 @@ export function rankAnswerEvidence( (a, b) => b.score - a.score || (b.result.hybrid_score ?? b.result.similarity ?? 0) - (a.result.hybrid_score ?? a.result.similarity ?? 0) || + (b.result.score_explanation?.rankScore ?? 0) - (a.result.score_explanation?.rankScore ?? 0) || a.index - b.index, ); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index e86d5c99a..5c9e068c4 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -549,6 +549,42 @@ function recordRetrievalLayer( } } +/** + * Keep distinct results in the current ranked order while upgrading duplicate chunks to the + * strongest released-hybrid copy. + * + * App-layer rank scores remain available to answer evidence ranking and telemetry, but the + * live corpus gate has not validated replacing the final distinct-result order with them. + */ +export function stabilizeReleasedSearchOrder(results: SearchResult[]) { + const compareReleasedSearchStrength = (left: SearchResult, right: SearchResult) => { + const leftHybrid = left.hybrid_score ?? left.similarity ?? 0; + const rightHybrid = right.hybrid_score ?? right.similarity ?? 0; + if (rightHybrid !== leftHybrid) return rightHybrid - leftHybrid; + const leftSimilarity = left.similarity ?? 0; + const rightSimilarity = right.similarity ?? 0; + if (rightSimilarity !== leftSimilarity) return rightSimilarity - leftSimilarity; + if (right.relevance?.score !== left.relevance?.score) + return (right.relevance?.score ?? 0) - (left.relevance?.score ?? 0); + return left.id.localeCompare(right.id); + }; + const strongestById = new Map(); + for (const result of results) { + const current = strongestById.get(result.id); + if (!current || compareReleasedSearchStrength(result, current) < 0) strongestById.set(result.id, result); + } + const deduped: SearchResult[] = []; + const seen = new Set(); + for (const result of results) { + if (seen.has(result.id)) continue; + seen.add(result.id); + deduped.push(strongestById.get(result.id) ?? result); + } + results.length = 0; + results.push(...deduped); + return results; +} + /** Record search score telemetry. */ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) { if (!results.length) { @@ -565,17 +601,7 @@ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchR return; } - // Preserve the rankScore ordering established by clinical/second-stage ranking. Coverage - // telemetry still uses the raw hybrid signal, but must never mutate the returned result order. - const deduped: SearchResult[] = []; - const seen = new Set(); - for (const result of results) { - if (seen.has(result.id)) continue; - seen.add(result.id); - deduped.push(result); - } - results.length = 0; - results.push(...deduped); + stabilizeReleasedSearchOrder(results); const coverageScores = results .map((result) => Math.max(0, result.hybrid_score ?? result.similarity ?? 0)) .sort((left, right) => right - left); diff --git a/tests/answer-ranking.test.ts b/tests/answer-ranking.test.ts index fd0a591a9..b7b0d2668 100644 --- a/tests/answer-ranking.test.ts +++ b/tests/answer-ranking.test.ts @@ -179,6 +179,26 @@ describe("answer evidence ranking", () => { ranking.scoresByChunkId.get("low-semantic") ?? 0, ); }); + + it("uses the app-layer rank only as a final answer-evidence tie-breaker", () => { + const lowerRank = result({ + id: "lower-rank", + content: "Lithium dosing guidance.", + hybrid_score: 0.7, + score_explanation: { rankScore: 0.8 } as NonNullable, + }); + const higherRank = result({ + id: "higher-rank", + content: "Lithium dosing guidance.", + hybrid_score: 0.7, + score_explanation: { rankScore: 1.1 } as NonNullable, + }); + + const ranking = rankAnswerEvidence("lithium dosing guidance", [lowerRank, higherRank]); + + expect(ranking.rankedResults.map((item) => item.id)).toEqual(["higher-rank", "lower-rank"]); + expect(ranking.scoresByChunkId.get("higher-rank")).toBe(ranking.scoresByChunkId.get("lower-rank")); + }); }); describe("high-yield answer bolding", () => { diff --git a/tests/rag-second-stage-ranking.test.ts b/tests/rag-second-stage-ranking.test.ts index 70d28484e..52c7dfe81 100644 --- a/tests/rag-second-stage-ranking.test.ts +++ b/tests/rag-second-stage-ranking.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { applySecondStageRerankIfNeeded } from "../src/lib/rag"; +import { applySecondStageRerankIfNeeded, stabilizeReleasedSearchOrder } from "../src/lib/rag"; import type { SearchTelemetry } from "../src/lib/rag-contracts"; import type { SearchResult, SearchScoreExplanation } from "../src/lib/types"; @@ -136,4 +136,54 @@ describe("second-stage rank score", () => { expect(ranked.map((item) => item.id)).toEqual(["dose-evidence-without-explanation", "supported"]); expect(ranked[1].score_explanation?.finalRank).toBe(2); }); + + it("keeps distinct rank order while upgrading duplicate chunks to the strongest released-hybrid copy", () => { + const telemetry = {} as SearchTelemetry; + const strongerHybridDuplicate = result({ + id: "stronger-hybrid", + document_id: "older-doc", + hybrid_score: 0.8, + score_explanation: explanation(0.4), + source_metadata: { + source_title: "Older guideline", + publisher: "Test publisher", + jurisdiction: "Australia/WA", + version: "1", + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "outdated", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }, + }); + const higherRankScore = result({ + id: "higher-rank-score", + document_id: "current-doc", + hybrid_score: 0.79, + score_explanation: explanation(0.5), + }); + const weakerDuplicate = { + ...strongerHybridDuplicate, + hybrid_score: 0.78, + score_explanation: explanation(0.41), + content: "Weaker duplicate", + }; + + const secondStage = applySecondStageRerankIfNeeded({ + queryClass: "medication_dose_risk", + results: [higherRankScore, weakerDuplicate, strongerHybridDuplicate], + telemetry, + topK: 3, + }); + expect(secondStage.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid", "stronger-hybrid"]); + + const stabilized = stabilizeReleasedSearchOrder(secondStage); + + expect(stabilized.map((item) => item.id)).toEqual(["higher-rank-score", "stronger-hybrid"]); + expect(stabilized[1].hybrid_score).toBe(0.8); + expect(stabilized[1].content).toBe(strongerHybridDuplicate.content); + }); });