Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions src/lib/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -549,17 +549,24 @@ function recordRetrievalLayer(
}
}

/** Whether a release-rank score is safe to use for bounded release ordering. */
function isBoundedReleaseRankScore(value: number | undefined): value is number {
return typeof value === "number" && Number.isFinite(value);
}

/** Whether the current result set carries bounded second-stage release scores. */
function resultsHaveReleaseRankScore(results: SearchResult[]) {
return results.some((result) => result.score_explanation?.releaseRankScore !== undefined);
return results.some((result) => isBoundedReleaseRankScore(result.score_explanation?.releaseRankScore));
}

/**
* Keep the released result order on the live-eval-proven hybrid and bounded second-stage signals.
*
* App-layer rank scores remain available to answer evidence ranking and telemetry, but the
* live corpus gate has not validated using them as the final retrieval order. Resolve duplicate
* chunks to their strongest released-hybrid copy before sorting the distinct results.
* chunks to their strongest released-hybrid copy. Distinct results keep the clinical selection's
* existing order unless `preferSecondStageScore` is enabled and the current set carries bounded
* second-stage release scores.
*/
export function stabilizeReleasedSearchOrder(results: SearchResult[], preferSecondStageScore = false) {
const useSecondStageReleaseOrder = preferSecondStageScore && resultsHaveReleaseRankScore(results);
Expand All @@ -575,9 +582,12 @@ export function stabilizeReleasedSearchOrder(results: SearchResult[], preferSeco
return left.id.localeCompare(right.id);
};
const compareReleasedSearchOrder = (left: SearchResult, right: SearchResult) => {
if (!useSecondStageReleaseOrder) return compareReleasedHybridStrength(left, right);
const leftReleaseScore = left.score_explanation?.releaseRankScore ?? left.hybrid_score ?? left.similarity ?? 0;
const rightReleaseScore = right.score_explanation?.releaseRankScore ?? right.hybrid_score ?? right.similarity ?? 0;
const leftReleaseScore = isBoundedReleaseRankScore(left.score_explanation?.releaseRankScore)
? left.score_explanation.releaseRankScore
: (left.hybrid_score ?? left.similarity ?? 0);
const rightReleaseScore = isBoundedReleaseRankScore(right.score_explanation?.releaseRankScore)
? right.score_explanation.releaseRankScore
: (right.hybrid_score ?? right.similarity ?? 0);
if (rightReleaseScore !== leftReleaseScore) return rightReleaseScore - leftReleaseScore;
const leftSimilarity = left.similarity ?? 0;
const rightSimilarity = right.similarity ?? 0;
Expand All @@ -591,13 +601,15 @@ export function stabilizeReleasedSearchOrder(results: SearchResult[], preferSeco
const current = strongestById.get(result.id);
if (!current || compareReleasedHybridStrength(result, current) < 0) strongestById.set(result.id, result);
}
const deduped = [...strongestById.values()]
.sort(compareReleasedSearchOrder)
.map((result, index) =>
result.score_explanation
? { ...result, score_explanation: { ...result.score_explanation, finalRank: index + 1 } }
: result,
);
const distinctResults = [...strongestById.values()];
const releasedResults = useSecondStageReleaseOrder
? distinctResults.sort(compareReleasedSearchOrder)
: distinctResults;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
const deduped = releasedResults.map((result, index) =>
result.score_explanation
? { ...result, score_explanation: { ...result.score_explanation, finalRank: index + 1 } }
: result,
);
results.length = 0;
results.push(...deduped);
return results;
Expand Down
41 changes: 35 additions & 6 deletions tests/rag-second-stage-ranking.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,18 +234,27 @@ describe("second-stage rank score", () => {
result({
id: "lithium-monitoring",
hybrid_score: 0.79,
content: "Lithium monitoring evidence selected first by the clinical layer.",
score_explanation: explanation(0.9),
}),
result({
id: "cardiac-dose",
hybrid_score: 0.8,
hybrid_score: 0.95,
score_explanation: explanation(0.5),
}),
result({
id: "lithium-monitoring",
hybrid_score: 0.85,
content: "Stronger released-hybrid copy of the lithium evidence.",
score_explanation: explanation(0.85),
}),
];

expect(stabilizeReleasedSearchOrder([...clinicallyOrdered], true).map((item) => item.id)).toEqual(
stabilizeReleasedSearchOrder([...clinicallyOrdered], false).map((item) => item.id),
);
const stabilized = stabilizeReleasedSearchOrder([...clinicallyOrdered], true);

expect(stabilized.map((item) => item.id)).toEqual(["lithium-monitoring", "cardiac-dose"]);
expect(stabilized[0].content).toBe("Stronger released-hybrid copy of the lithium evidence.");
expect(stabilized.map((item) => item.score_explanation?.finalRank)).toEqual([1, 2]);
});

it("only applies bounded release ordering when the current result set carries releaseRankScore", () => {
Expand All @@ -264,8 +273,28 @@ describe("second-stage rank score", () => {
"release-ranked",
"hybrid-leading",
]);
expect(stabilizeReleasedSearchOrder([withoutReleaseScore, withReleaseScore], false).map((item) => item.id)).toEqual(
["hybrid-leading", "release-ranked"],
expect(stabilizeReleasedSearchOrder([withReleaseScore, withoutReleaseScore], false).map((item) => item.id)).toEqual(
["release-ranked", "hybrid-leading"],
);
});

it("preserves clinical order when releaseRankScore values are not finite", () => {
const clinicallyOrdered = [
result({
id: "first",
hybrid_score: 0.7,
score_explanation: { ...explanation(0.7), releaseRankScore: Number.NaN },
}),
result({
id: "second",
hybrid_score: 0.9,
score_explanation: { ...explanation(0.9), releaseRankScore: Number.POSITIVE_INFINITY },
}),
];

expect(stabilizeReleasedSearchOrder([...clinicallyOrdered], true).map((item) => item.id)).toEqual([
"first",
"second",
]);
});
});