diff --git a/.env.example b/.env.example index cdf6c9c01..ac2d9744e 100644 --- a/.env.example +++ b/.env.example @@ -82,6 +82,11 @@ RAG_AWAIT_QUERY_LOGS=false #NEXT_PUBLIC_MOCKUPS_ENABLED=false # Persist raw clinical query text in logs. Default off; blocked in production readiness. #RAG_PERSIST_RAW_QUERY_TEXT=false +# Persist the full generated answer text (rag_queries.answer and promoted eval-case +# metadata). It is PHI-derived and can restate patient specifics, so it is dropped at +# rest by default. The offline eval pipeline reads the in-memory answer, not this +# column. Default off; blocked in production readiness (PIA-3). +#RAG_PERSIST_ANSWER_TEXT=false # Server-side key for the redacted query-hash placeholder (min 16 chars). # When set, stored query hashes are HMAC-SHA256 keyed pseudonyms — not # offline-reversible and not correlatable outside this deployment. REQUIRED in diff --git a/docs/privacy-impact-assessment.md b/docs/privacy-impact-assessment.md index fae5b515f..19b1a65bc 100644 --- a/docs/privacy-impact-assessment.md +++ b/docs/privacy-impact-assessment.md @@ -42,15 +42,15 @@ material. **Top gaps (full register in §10):** -| ID | Risk | One-line | -| ----- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| PIA-1 | High | Cross-border disclosure to OpenAI (US) has no code-visible DPA/ZDR and no user-facing notice/consent (APP 8, APP 5). | -| PIA-2 | High | Query-hash HMAC downgrades to **unsalted, dictionary-reversible SHA-256** if `RAG_QUERY_HASH_SECRET` is unset — nothing enforces it in production. | -| PIA-3 | Medium | Generated **answer text is stored un-redacted** in `rag_queries`, and can restate the patient details from the query. | -| PIA-4 | Medium | `rag_query_misses` has **no retention/purge job** (only `rag_queries` and `rag_retrieval_logs` do). | -| PIA-5 | Medium | No privacy policy / collection notice / data-handling documentation shipped (APP 1, APP 5). | -| PIA-6 | Low-Med | OpenAI **prompt-cache retention is forced to 24h** for gpt-5.5 regardless of config — query + retrieved excerpts persist ≤24h at OpenAI. | -| PIA-7 | Low | `RAG_PERSIST_RAW_QUERY_TEXT=true` would store raw PHI query text with no secondary safeguard beyond the 30-day purge. | +| ID | Risk | One-line | +| ----- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| PIA-1 | High | Cross-border disclosure to OpenAI (US) has no code-visible DPA/ZDR and no user-facing notice/consent (APP 8, APP 5). | +| PIA-2 | High | Query-hash HMAC downgrades to **unsalted, dictionary-reversible SHA-256** if `RAG_QUERY_HASH_SECRET` is unset — nothing enforces it in production. | +| PIA-3 | Resolved | Generated answer text is no longer persisted to `rag_queries` by default; answer-text persistence is gated behind `RAG_PERSIST_ANSWER_TEXT` (default off, blocked in production readiness). | +| PIA-4 | Medium | `rag_query_misses` has **no retention/purge job** (only `rag_queries` and `rag_retrieval_logs` do). | +| PIA-5 | Medium | No privacy policy / collection notice / data-handling documentation shipped (APP 1, APP 5). | +| PIA-6 | Low-Med | OpenAI **prompt-cache retention is forced to 24h** for gpt-5.5 regardless of config — query + retrieved excerpts persist ≤24h at OpenAI. | +| PIA-7 | Low | `RAG_PERSIST_RAW_QUERY_TEXT=true` would store raw PHI query text with no secondary safeguard beyond the 30-day purge. | --- @@ -60,7 +60,7 @@ material. | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------ | | Clinical reference corpus (documents, chunks, embeddings, images, tables) | Supabase (Sydney) + storage buckets | Low–Medium | Published guidelines are not PHI; **uploaded** docs _could_ contain PHI. | | Free-text clinical queries | Transient in request; hashed into `rag_queries` / `rag_query_misses` / `rag_retrieval_logs`; sent to OpenAI (US) | **High (potential PHI)** | The primary incidental-PHI vector. | -| Generated answers | `rag_queries.answer`, `rag_response_cache.payload` | **High (derived from PHI query + corpus)** | Stored un-redacted; see PIA-3. | +| Generated answers | `rag_queries.answer` (not persisted unless `RAG_PERSIST_ANSWER_TEXT`), `rag_response_cache.payload` | **High (derived from PHI query + corpus)** | Answer text dropped at rest by default; see PIA-3. | | User identity | Supabase Auth (`auth.users`), `owner_id` foreign keys | Medium (PII) | Email + SSO identity; managed by Supabase Auth. | | Audit trail | `audit_logs` | Medium | Append-only, service-role-only, retained indefinitely by design. | | Operational telemetry | `rag_retrieval_logs`, ingestion job tables | Low–Medium | Redacted query text; per-owner. | @@ -105,7 +105,7 @@ Clinician browser │ insertRagQuery(): rag.ts:1983 │ • query = **hash placeholder** (queryTextForStorage) ← redacted │ • normalized_query= **hash placeholder** ← redacted - │ • **answer = full generated text** ← NOT redacted (PIA-3) + │ • answer = null unless RAG_PERSIST_ANSWER_TEXT ← dropped at rest (PIA-3) │ • source_chunk_ids= real chunk UUIDs ← owner's own data │ • metadata.query_hash = HMAC/SHA-256 (query-privacy.ts:51) │ @@ -178,12 +178,12 @@ privacy item to close before real patient use (PIA-1). All three log tables are **owner-stamped** and **RLS-enabled** (owner-read for authenticated users; service-role for writes). Redaction is applied centrally at every write site. -| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | -| -------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | --------------------------------------------------------- | -| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts:33-39](src/lib/query-privacy.ts)); write at [rag.ts:1983-1994](src/lib/rag.ts) | **`answer` stored un-redacted** (PIA-3); `source_chunk_ids` (own data) | owner-read, [schema.sql:3932](supabase/schema.sql) | -| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts:558-559](src/app/api/search/route.ts), [interaction/route.ts:88-89](src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql:3935](supabase/schema.sql) | -| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts:556-559](src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql:3938](supabase/schema.sql) | -| `audit_logs` | N/A (no query text) | action/resource metadata only; error strings pass through `redactLogValue` ([privacy.ts:5-31](src/lib/privacy.ts)) | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql:3959](supabase/schema.sql) | +| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | +| -------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts](src/lib/query-privacy.ts)); write via `insertRagQuery` ([rag.ts](src/lib/rag.ts)) | `answer` dropped at rest unless `RAG_PERSIST_ANSWER_TEXT` (`answerTextForStorage`, PIA-3); `source_chunk_ids` (own data) | owner-read, [schema.sql:3932](supabase/schema.sql) | +| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts:558-559](src/app/api/search/route.ts), [interaction/route.ts:88-89](src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql:3935](supabase/schema.sql) | +| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts:556-559](src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql:3938](supabase/schema.sql) | +| `audit_logs` | N/A (no query text) | action/resource metadata only; error strings pass through `redactLogValue` ([privacy.ts:5-31](src/lib/privacy.ts)) | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql:3959](supabase/schema.sql) | ### 5.1 M15 HMAC query-hash fix — verified present, conditionally active @@ -301,19 +301,20 @@ the operative framework for a WA private clinician. (The WA _Privacy and Respons Act 2024_ targets WA **public-sector** entities and may apply to public-health deployments — confirm with counsel if this is deployed inside a WA Health service.) -| APP | Obligation | Status in this app | Gap | -| ----------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------- | -| **APP 1** — open & transparent management | Have a clear, up-to-date APP privacy policy | No privacy policy/collection notice ships in the repo | PIA-5 | -| **APP 3** — collection of sensitive info | Collect health info only with consent + where reasonably necessary | App does not solicit PHI; incidental via free-text. No consent gate/notice on the query box | PIA-5 | -| **APP 5** — notification of collection | Tell individuals what's collected & disclosed (incl. overseas) | No collection notice; OpenAI disclosure not surfaced to the clinician | PIA-1, PIA-5 | -| **APP 6** — use/disclosure | Use only for the primary purpose or a permitted secondary purpose | Query used for answer generation (primary). Log retention = quality/eval (secondary) — defensible but should be documented | PIA-5 | -| **APP 8** — cross-border disclosure | Discloser stays accountable for the overseas recipient unless an exception applies | Disclosure to OpenAI (US); no code-visible DPA/ZDR; accountability unclear | **PIA-1** | -| **APP 11** — security & destruction | Reasonable security; destroy/de-identify when no longer needed | Strong: Sydney residency, RLS, private storage, query hashing, 30/90-day purge. Weakened by PIA-2 (conditional HMAC), PIA-3 (un-redacted answers), PIA-4 (misses never purged) | PIA-2/3/4 | -| **NDB scheme** (Pt IIIC) | Notify OAIC + individuals of eligible breaches of health info | No documented breach-response runbook tied to these tables | Recommend adding | +| APP | Obligation | Status in this app | Gap | +| ----------------------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| **APP 1** — open & transparent management | Have a clear, up-to-date APP privacy policy | No privacy policy/collection notice ships in the repo | PIA-5 | +| **APP 3** — collection of sensitive info | Collect health info only with consent + where reasonably necessary | App does not solicit PHI; incidental via free-text. No consent gate/notice on the query box | PIA-5 | +| **APP 5** — notification of collection | Tell individuals what's collected & disclosed (incl. overseas) | No collection notice; OpenAI disclosure not surfaced to the clinician | PIA-1, PIA-5 | +| **APP 6** — use/disclosure | Use only for the primary purpose or a permitted secondary purpose | Query used for answer generation (primary). Log retention = quality/eval (secondary) — defensible but should be documented | PIA-5 | +| **APP 8** — cross-border disclosure | Discloser stays accountable for the overseas recipient unless an exception applies | Disclosure to OpenAI (US); no code-visible DPA/ZDR; accountability unclear | **PIA-1** | +| **APP 11** — security & destruction | Reasonable security; destroy/de-identify when no longer needed | Strong: Sydney residency, RLS, private storage, query hashing, 30/90-day purge, answer text dropped at rest by default (PIA-3 closed). Weakened by PIA-2 (conditional HMAC), PIA-4 (misses never purged) | PIA-2/4 | +| **NDB scheme** (Pt IIIC) | Notify OAIC + individuals of eligible breaches of health info | No documented breach-response runbook tied to these tables | Recommend adding | **Overall:** the _engineering_ controls for data-at-rest are strong and largely APP-11-aligned. The -material shortfalls are **governance/contractual** (APP 8 cross-border, APP 1/5 notices) plus two -**hardening** items (conditional HMAC, un-redacted answer retention). None of these are cross-tenant +material shortfalls are **governance/contractual** (APP 8 cross-border, APP 1/5 notices) plus the +**hardening** item still open (conditional HMAC); un-redacted answer retention (PIA-3) is now closed — +answer text is dropped at rest by default. None of these are cross-tenant data-leak bugs — the tenancy review found **zero** confirmed cross-tenant leaks — they are compliance-posture and PHI-minimisation gaps. @@ -343,15 +344,22 @@ compliance-posture and PHI-minimisation gaps. `requireServerEnv` pattern) when `NODE_ENV=production` and the secret is missing. Set it on the live project now. -### PIA-3 — Generated answers stored un-redacted in `rag_queries` **(Medium)** - -- **Risk:** The `answer` column holds the full generated text, which can restate patient specifics - echoed from the query; the query itself is hashed but the answer is not. Owner-scoped (not - cross-tenant) and purged at 30 days, but it is un-redacted PHI-derived content at rest. -- **Evidence:** answer-path insert stores `answer` verbatim ([rag.ts:7580-7584](src/lib/rag.ts), traced). -- **Fix:** Gate answer-text persistence behind the same `RAG_PERSIST_RAW_QUERY_TEXT` flag (or a new - `RAG_PERSIST_ANSWER_TEXT`), defaulting off; store only what eval/quality actually needs. Confirm the - eval pipeline's real requirement before changing. +### PIA-3 — Generated answers stored un-redacted in `rag_queries` **(Resolved)** + +- **Risk:** The `answer` column held the full generated text, which can restate patient specifics + echoed from the query; the query itself is hashed but the answer was not. Owner-scoped (not + cross-tenant) and purged at 30 days, but it was un-redacted PHI-derived content at rest. +- **Fix (shipped):** Answer-text persistence is gated behind a dedicated `RAG_PERSIST_ANSWER_TEXT` + flag (default **off**), applied centrally in `insertRagQuery` via `answerTextForStorage` + ([query-privacy.ts](src/lib/query-privacy.ts), [rag.ts](src/lib/rag.ts)) so every `logRagQuery` + caller is covered, and at the promoted-eval-case write in + [eval-cases/route.ts](src/app/api/eval-cases/route.ts). With the flag off the column is written as + `null` and each row records `metadata.answer_retained = false`. The offline eval/quality pipeline + reads the in-memory answer (`logQuery: false`) and never reads this column back + ([scripts/eval-rag.ts](scripts/eval-rag.ts), [scripts/eval-answer-quality.ts](scripts/eval-answer-quality.ts), + [scripts/promote-query-misses.ts](scripts/promote-query-misses.ts)), so persistence-off does not + affect eval — confirming the pipeline has no real dependency on stored answer text. The flag is + additionally blocked in a production-like environment by `npm run check:production-readiness`. ### PIA-4 — `rag_query_misses` never purged **(Medium)** @@ -388,9 +396,9 @@ compliance-posture and PHI-minimisation gaps. ## 11. Recommendation Before the app is used with real patients in a WA clinical setting, close **PIA-1** and **PIA-2** as -launch-blockers (cross-border contractual basis + user notice; mandatory HMAC secret), then **PIA-3/4** -as fast follow-ups (minimise answer retention; purge `rag_query_misses`), and ship the **PIA-5** -privacy documentation. The data-at-rest security posture (Sydney residency, RLS, private storage, +launch-blockers (cross-border contractual basis + user notice; mandatory HMAC secret). **PIA-3** is +closed (answer text is no longer persisted by default; gated behind `RAG_PERSIST_ANSWER_TEXT`); **PIA-4** +remains as a fast follow-up (purge `rag_query_misses`), and ship the **PIA-5** privacy documentation. The data-at-rest security posture (Sydney residency, RLS, private storage, query hashing, automated purge) is already strong and should be highlighted in the privacy policy as evidence of "reasonable steps" under APP 11. diff --git a/scripts/production-readiness.ts b/scripts/production-readiness.ts index a63b05d3a..c2f2db5a8 100644 --- a/scripts/production-readiness.ts +++ b/scripts/production-readiness.ts @@ -108,6 +108,15 @@ function recordRawQueryPersistenceProductionCheck() { } } +function recordAnswerPersistenceProductionCheck() { + if ( + (process.env.NODE_ENV === "production" || process.env.VERCEL_ENV === "production") && + process.env.RAG_PERSIST_ANSWER_TEXT === "true" + ) { + result.failures.push("RAG_PERSIST_ANSWER_TEXT=true is not allowed in a production-like environment."); + } +} + async function checkFileForServiceRoleExposure() { const envFiles = [".env", ".env.production", ".env.development"]; for (const fileName of envFiles) { @@ -132,6 +141,7 @@ async function main() { recordNoAuthProductionCheck(); recordDemoModeProductionCheck(); recordRawQueryPersistenceProductionCheck(); + recordAnswerPersistenceProductionCheck(); await checkFileForServiceRoleExposure(); if (!(await checkRequiredFile(path.join(process.cwd(), "package-lock.json"), "package-lock.json is required"))) { diff --git a/src/app/api/eval-cases/route.ts b/src/app/api/eval-cases/route.ts index 044ae0c3d..e75455861 100644 --- a/src/app/api/eval-cases/route.ts +++ b/src/app/api/eval-cases/route.ts @@ -5,6 +5,8 @@ import { clinicalQueryModeSchema } from "@/lib/clinical-query-mode"; import { env, isDemoMode } from "@/lib/env"; import { jsonError, PublicApiError } from "@/lib/http"; import { + answerPrivacyMetadata, + answerTextForStorage, normalizedQueryTextForStorage, queryDerivedTokensForStorage, queryPrivacyMetadata, @@ -164,7 +166,10 @@ export async function POST(request: Request) { rating, feedback_type: parsed.feedbackType ?? null, note: env.RAG_PERSIST_RAW_QUERY_TEXT ? parsed.note : null, - answer: env.RAG_PERSIST_RAW_QUERY_TEXT ? parsed.answer : null, + // PIA-3: the promoted answer is generated clinical text — gate it on the + // dedicated answer-retention flag, not the raw-query flag, so one switch + // governs answer-text persistence everywhere. + answer: answerTextForStorage(parsed.answer), query_class: parsed.queryClass ?? null, query_mode: parsed.queryMode, filters: parsed.filters ?? {}, @@ -174,6 +179,7 @@ export async function POST(request: Request) { cited_chunk_ids_rejected: parsed.citedChunkIds.length - citedChunkIds.length, captured_at: new Date().toISOString(), ...queryPrivacyMetadata(parsed.query), + ...answerPrivacyMetadata(), }, }) .select("id") diff --git a/src/lib/env.ts b/src/lib/env.ts index ae308cd6a..d4e22baa7 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -111,6 +111,17 @@ const envSchema = z.object({ .enum(["true", "false"]) .default("false") .transform((value) => value === "true"), + // PIA-3: rag_queries.answer holds the full generated answer text, which can + // restate patient specifics echoed from the query (the query is hashed, the + // answer is not). Default OFF: do not persist generated answer text at rest. + // The offline eval/quality pipeline reads the in-memory answer (logQuery:false) + // and never reads this column back, so persistence-off is safe. Set true only + // where retaining answer text is permitted and a retention policy exists + // (owner-scoped + 30-day purge). Blocked in production readiness. + RAG_PERSIST_ANSWER_TEXT: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true"), // Audit M15: server-side key for the redacted query hash. When set, stored // query hashes are HMAC-SHA256 (not offline-reversible, not correlatable // outside this deployment). When unset, the legacy unsalted SHA-256 is kept diff --git a/src/lib/query-privacy.ts b/src/lib/query-privacy.ts index 02f8d2d2a..a07d2cca6 100644 --- a/src/lib/query-privacy.ts +++ b/src/lib/query-privacy.ts @@ -39,6 +39,22 @@ export function normalizedQueryTextForStorage(query: string): string { return env.RAG_PERSIST_RAW_QUERY_TEXT ? normalizeQueryText(query) : queryHashStorageText(query); } +// PIA-3: the generated answer is stored verbatim in rag_queries.answer (and in +// promoted-eval-case metadata), so it can restate patient specifics echoed from +// the query. Unless answer retention is explicitly enabled, drop the answer text +// at rest and keep only the row's non-PHI telemetry (source chunk ids, model, +// hash metadata). The column is nullable, so null is the valid "not retained" +// marker. This is the single chokepoint governing answer-text persistence. +export function answerTextForStorage(answer: string | null | undefined): string | null { + return env.RAG_PERSIST_ANSWER_TEXT ? (answer ?? null) : null; +} + +// Privacy metadata to fold into a persisted row that carries an answer: records +// whether the generated answer text was retained, mirroring raw_query_retained. +export function answerPrivacyMetadata() { + return { answer_retained: env.RAG_PERSIST_ANSWER_TEXT }; +} + export function queryCacheKeyForStorage(cacheKey: string): string { return env.RAG_PERSIST_RAW_QUERY_TEXT ? cacheKey : `redacted-cache:${hashQueryText(cacheKey)}`; } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index b4d11feeb..eabb8ac03 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -102,7 +102,12 @@ import { } from "@/lib/clinical-search"; import { env, requestedOpenAIAnswerModels } from "@/lib/env"; import { logger } from "@/lib/logger"; -import { queryPrivacyMetadata, queryTextForStorage } from "@/lib/query-privacy"; +import { + answerPrivacyMetadata, + answerTextForStorage, + queryPrivacyMetadata, + queryTextForStorage, +} from "@/lib/query-privacy"; import { normalizeSourceMetadata } from "@/lib/source-metadata"; import { isReviewedTablePromotable } from "@/lib/table-review"; import { isClinicalImageEvidence, normalizeImageBbox } from "@/lib/image-filtering"; @@ -1299,11 +1304,18 @@ async function insertRagQuery(row: RagQueryInsert) { const supabase = createAdminClient(); // Redact potential-PHI raw query text centrally so every logRagQuery caller is // covered, and fold a stable hash + retention flag into metadata (RET-H4). + // The generated answer can restate patient specifics, so it is dropped at rest + // unless answer retention is explicitly enabled (PIA-3, default off). const rawQuery = typeof row.query === "string" ? row.query : ""; const safeRow = { ...row, query: queryTextForStorage(rawQuery), - metadata: { ...(row.metadata ?? {}), ...queryPrivacyMetadata(rawQuery) } as Json, + answer: answerTextForStorage(row.answer), + metadata: { + ...(row.metadata ?? {}), + ...queryPrivacyMetadata(rawQuery), + ...answerPrivacyMetadata(), + } as Json, }; await supabase.from("rag_queries").insert(safeRow); } diff --git a/tests/eval-cases-route.test.ts b/tests/eval-cases-route.test.ts index 7fb050b9b..fe30e6ab1 100644 --- a/tests/eval-cases-route.test.ts +++ b/tests/eval-cases-route.test.ts @@ -70,7 +70,10 @@ afterEach(() => { }); function mockEnv(overrides: Record = {}) { - return { isDemoMode: () => false, env: { RAG_PERSIST_RAW_QUERY_TEXT: false, ...overrides } }; + return { + isDemoMode: () => false, + env: { RAG_PERSIST_RAW_QUERY_TEXT: false, RAG_PERSIST_ANSWER_TEXT: false, ...overrides }, + }; } describe("/api/eval-cases", () => { @@ -163,8 +166,43 @@ describe("/api/eval-cases", () => { expect(serialized).not.toContain("clozapine"); }); - it("retains raw query and answer when RAG_PERSIST_RAW_QUERY_TEXT is true", async () => { + it("retains raw query and answer when both retention flags are enabled", async () => { const { client, insert } = createInsertMock(); + vi.doMock("@/lib/env", () => mockEnv({ RAG_PERSIST_RAW_QUERY_TEXT: true, RAG_PERSIST_ANSWER_TEXT: true })); + vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => client })); + vi.doMock("@/lib/supabase/auth", () => ({ + AuthenticationError: class AuthenticationError extends Error {}, + requireAuthenticatedUser: vi.fn(async () => ({ id: userId })), + unauthorizedResponse: () => Response.json({ error: "Authentication required." }, { status: 401 }), + })); + const { POST } = await import("../src/app/api/eval-cases/route"); + + const response = await POST( + request({ + query: "What monitoring is needed for clozapine?", + rating: "good", + answer: "Monitor FBC.", + queryMode: "auto", + queryClass: "table_threshold", + sourceChunkIds: [], + citedChunkIds: [], + }), + ); + const payload = insert.mock.calls[0]?.[0] as Record; + + expect(response.status).toBe(201); + expect(payload).toMatchObject({ query: "What monitoring is needed for clozapine?" }); + expect(payload.metadata).toMatchObject({ + answer: "Monitor FBC.", + raw_query_retained: true, + answer_retained: true, + }); + }); + + it("gates answer retention on RAG_PERSIST_ANSWER_TEXT independently of the raw-query flag (PIA-3)", async () => { + const { client, insert } = createInsertMock(); + // Raw query retention on, answer retention off: the query text is kept but the + // generated answer is still dropped — the two are decoupled. vi.doMock("@/lib/env", () => mockEnv({ RAG_PERSIST_RAW_QUERY_TEXT: true })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => client })); vi.doMock("@/lib/supabase/auth", () => ({ @@ -189,7 +227,12 @@ describe("/api/eval-cases", () => { expect(response.status).toBe(201); expect(payload).toMatchObject({ query: "What monitoring is needed for clozapine?" }); - expect(payload.metadata).toMatchObject({ answer: "Monitor FBC.", raw_query_retained: true }); + expect(payload.metadata).toMatchObject({ + answer: null, + raw_query_retained: true, + answer_retained: false, + }); + expect(JSON.stringify(payload.metadata)).not.toContain("Monitor FBC."); }); it("captures a needs-fixing answer without requiring expected UUID fields", async () => { diff --git a/tests/privacy.test.ts b/tests/privacy.test.ts index ce9edd29d..bc509828a 100644 --- a/tests/privacy.test.ts +++ b/tests/privacy.test.ts @@ -147,6 +147,32 @@ describe("query privacy storage helpers", () => { }); }); +describe("answer persistence storage helper (PIA-3)", () => { + it("drops the generated answer text at rest by default", async () => { + vi.doMock("@/lib/env", () => ({ env: { RAG_PERSIST_ANSWER_TEXT: false } })); + const { answerTextForStorage, answerPrivacyMetadata } = await import("../src/lib/query-privacy"); + + // A generated answer can restate patient specifics echoed from the query. + const phiAnswer = "Jane Citizen (MRN 123456) should have clozapine withheld below an ANC of 1.5."; + expect(answerTextForStorage(phiAnswer)).toBeNull(); + expect(answerTextForStorage("")).toBeNull(); + expect(answerTextForStorage(null)).toBeNull(); + expect(answerTextForStorage(undefined)).toBeNull(); + expect(answerPrivacyMetadata()).toEqual({ answer_retained: false }); + }); + + it("retains the answer text only when answer retention is explicitly enabled", async () => { + vi.doMock("@/lib/env", () => ({ env: { RAG_PERSIST_ANSWER_TEXT: true } })); + const { answerTextForStorage, answerPrivacyMetadata } = await import("../src/lib/query-privacy"); + + expect(answerTextForStorage("Monitor FBC weekly for the first 18 weeks.")).toBe( + "Monitor FBC weekly for the first 18 weeks.", + ); + expect(answerTextForStorage(null)).toBeNull(); + expect(answerPrivacyMetadata()).toEqual({ answer_retained: true }); + }); +}); + describe("queryVocabularyAliasesForStorage (RET-H4-safe candidate aliases)", () => { it("returns only curated vocabulary canonicals matched by the query, never query text", async () => { vi.doMock("@/lib/env", () => ({ env: { RAG_PERSIST_RAW_QUERY_TEXT: false } })); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 1a1a0d9de..43f6386e6 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -828,10 +828,19 @@ describe("RAG structured-output fallback", () => { expect(answer.routingReason).toContain("strong_quality_retry"); expect(answer.openAIRequestIds).toEqual(["req_fast_template", "req_strong_template", "req_strong_quality"]); expect(insert).toHaveBeenCalledTimes(1); - const insertCalls = insert.mock.calls as unknown as Array<[{ metadata?: Record }]>; - const loggedMetadata = insertCalls[0]?.[0]?.metadata ?? {}; + const insertCalls = insert.mock.calls as unknown as Array< + [{ answer?: unknown; metadata?: Record }] + >; + const loggedRow = insertCalls[0]?.[0] ?? {}; + const loggedMetadata = loggedRow.metadata ?? {}; expect(loggedMetadata.answer_retry_count).toBe(2); expect(loggedMetadata.answer_retry_reasons).toEqual(["fast_template_retry_strong", "strong_quality_retry"]); + // PIA-3: the generated answer text must not be persisted to rag_queries.answer + // unless RAG_PERSIST_ANSWER_TEXT is enabled (default off), and the row records + // that the answer was not retained. + expect(loggedRow.answer).toBeNull(); + expect(loggedMetadata.answer_retained).toBe(false); + expect(JSON.stringify(loggedRow)).not.toContain("review intervals"); }); it("filters table-caption metadata from extractive answer points", async () => {