Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/lib/answer-render-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,16 @@ function deriveTrust(answer: RagAnswer): AnswerRenderTrust {
}

if (retrievalBlocked || !sourceBacked || hasFaithfulnessWarning || answer.confidence === "low") return "low";
const highRiskSupport = (answer.supportedClaims ?? []).filter((claim) => claim.riskClass === "high_risk");
const highRiskAuthorityAccepted = highRiskSupport.every(
// D5 (audit item, ships OFF): when enabled, unverified-authority evidence caps
// trust for ALL supported claims, not just high-risk ones. Clinical product
// decision — flip only behind a green golden answer-quality eval. This module
// renders client-side, so the flag is a NEXT_PUBLIC build-time inline (unset
// = false = existing high-risk-only behavior).
const capAllClaims = process.env.NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS === "true";
const authorityGatedClaims = capAllClaims
? (answer.supportedClaims ?? [])
: (answer.supportedClaims ?? []).filter((claim) => claim.riskClass === "high_risk");
const authorityAccepted = authorityGatedClaims.every(
(claim) =>
claim.supportStatus === "direct" &&
claim.supportingChunkIds.length > 0 &&
Expand All @@ -166,7 +174,7 @@ function deriveTrust(answer: RagAnswer): AnswerRenderTrust {
return authority === "approved" || authority === "locally_reviewed";
}),
);
if (highRiskSupport.length > 0 && !highRiskAuthorityAccepted) return "medium";
if (authorityGatedClaims.length > 0 && !authorityAccepted) return "medium";
if (answer.confidence === "high") return "high";
return "medium";
}
Expand Down
2 changes: 2 additions & 0 deletions src/lib/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,8 @@ function secondStageScore(result: SearchResult, queryClass: RagQueryClass | unde
Math.max(0, sourceQuality - w.visualIntelligencePivot) * w.visualIntelligenceSlope,
);
if (result.source_metadata?.document_status === "outdated") score -= w.outdatedPenalty;
// D4: ships 0 (no-op) — activate via RAG_RANKING_CONFIG only behind a green golden eval.
if (result.source_metadata?.document_status === "unknown") score -= w.unknownCurrentnessPenalty;
if (result.source_metadata?.extraction_quality === "poor") score -= w.poorExtractionPenalty;
if (
result.indexing_quality?.quality_score !== undefined &&
Expand Down
12 changes: 12 additions & 0 deletions src/lib/ranking-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ export type SecondStageWeights = {
visualIntelligenceSlope: number;
/** Penalties for governance/quality signals. */
outdatedPenalty: number;
/**
* Penalty for document_status "unknown" (currentness never established).
* Ships 0 = OFF (audit item D4): per the #118 lesson, governance weighting
* needs golden-eval proof before activation — enable via RAG_RANKING_CONFIG
* only behind a fresh green golden retrieval + answer-quality run.
*/
unknownCurrentnessPenalty: number;
poorExtractionPenalty: number;
lowIndexQualityPenalty: number;
lowIndexQualityThreshold: number;
Expand Down Expand Up @@ -77,6 +84,7 @@ export const defaultRankingConfig: RankingConfig = {
visualIntelligencePivot: 0.55,
visualIntelligenceSlope: 0.08,
outdatedPenalty: 0.035,
unknownCurrentnessPenalty: 0,
poorExtractionPenalty: 0.035,
lowIndexQualityPenalty: 0.035,
lowIndexQualityThreshold: 0.55,
Expand Down Expand Up @@ -137,6 +145,10 @@ export function resolveRankingConfig(raw?: string | null): RankingConfig {
visualIntelligencePivot: num(ss.visualIntelligencePivot, d.secondStage.visualIntelligencePivot),
visualIntelligenceSlope: num(ss.visualIntelligenceSlope, d.secondStage.visualIntelligenceSlope),
outdatedPenalty: num(ss.outdatedPenalty, d.secondStage.outdatedPenalty),
unknownCurrentnessPenalty: Math.max(
0,
num(ss.unknownCurrentnessPenalty, d.secondStage.unknownCurrentnessPenalty),
),
poorExtractionPenalty: num(ss.poorExtractionPenalty, d.secondStage.poorExtractionPenalty),
lowIndexQualityPenalty: num(ss.lowIndexQualityPenalty, d.secondStage.lowIndexQualityPenalty),
lowIndexQualityThreshold: num(ss.lowIndexQualityThreshold, d.secondStage.lowIndexQualityThreshold),
Expand Down
60 changes: 59 additions & 1 deletion tests/answer-render-policy.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import {
buildAnswerRenderModel,
describeSourceStrengthForCopy,
Expand Down Expand Up @@ -304,6 +304,64 @@ describe("answer render policy", () => {
expect(model.trust).not.toBe("high");
});

it("keeps high trust for routine claims on unverified evidence while the D5 flag is off", () => {
// Locks the zero-change default: only high-risk claims are authority-gated.
const model = buildAnswerRenderModel(
answer({
supportedClaims: [
{
claimId: "claim-1",
text: "Document the review date.",
riskClass: "routine",
supportingChunkIds: ["chunk-1"],
supportStatus: "direct",
},
],
evidenceAssessments: {
"chunk-1": {
relevance: "direct",
claimSupport: "direct",
authority: "unverified",
currency: "current",
extractionQuality: "good",
},
},
}),
);
expect(model.trust).toBe("high");
});

it("caps trust for ANY claim on unverified evidence when NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS is on (D5)", () => {
vi.stubEnv("NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS", "true");
try {
const model = buildAnswerRenderModel(
answer({
supportedClaims: [
{
claimId: "claim-1",
text: "Document the review date.",
riskClass: "routine",
supportingChunkIds: ["chunk-1"],
supportStatus: "direct",
},
],
evidenceAssessments: {
"chunk-1": {
relevance: "direct",
claimSupport: "direct",
authority: "unverified",
currency: "current",
extractionQuality: "good",
},
},
}),
);
expect(model.trust).toBe("medium");
} finally {
vi.unstubAllEnvs();
}
});

it.each([
["missing assessment", undefined],
[
Expand Down
11 changes: 11 additions & 0 deletions tests/ranking-config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ describe("ranking-config defaults (W6 — zero behavior change)", () => {
expect(w.visualIntelligencePivot).toBe(0.55);
expect(w.visualIntelligenceSlope).toBe(0.08);
expect(w.outdatedPenalty).toBe(0.035);
// D4 ships OFF — unknown-currentness weighting needs golden-eval proof (#118 lesson).
expect(w.unknownCurrentnessPenalty).toBe(0);
expect(w.poorExtractionPenalty).toBe(0.035);
expect(w.lowIndexQualityPenalty).toBe(0.035);
expect(w.lowIndexQualityThreshold).toBe(0.55);
Expand Down Expand Up @@ -58,6 +60,15 @@ describe("resolveRankingConfig override merge", () => {
// Untouched weights fall back to defaults.
expect(cfg.secondStage.positionBase).toBe(0.09);
expect(cfg.documentDiversityPenalty).toBe(0.03);
// D4 activation path: the JSON override can set the penalty; negatives clamp to 0.
expect(
resolveRankingConfig(JSON.stringify({ secondStage: { unknownCurrentnessPenalty: 0.03 } })).secondStage
.unknownCurrentnessPenalty,
).toBe(0.03);
expect(
resolveRankingConfig(JSON.stringify({ secondStage: { unknownCurrentnessPenalty: -1 } })).secondStage
.unknownCurrentnessPenalty,
).toBe(0);
});

it("ignores non-numeric values and clamps diversity penalties to non-negative", () => {
Expand Down