diff --git a/package.json b/package.json index 49f795cdf..a8f6a3bed 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "prompt-for-codex-medical-knowledge-base", "version": "0.1.0", "private": true, - "packageManager": "npm@10.9.8", + "packageManager": "npm@11.12.1", "engines": { "node": "24.x", "npm": "11.x" diff --git a/src/app/api/documents/[id]/signed-url/route.ts b/src/app/api/documents/[id]/signed-url/route.ts index ce55e7b40..a950b99db 100644 --- a/src/app/api/documents/[id]/signed-url/route.ts +++ b/src/app/api/documents/[id]/signed-url/route.ts @@ -1,14 +1,16 @@ import { NextResponse } from "next/server"; +import { z } from "zod"; import { getDemoDocument } from "@/lib/demo-data"; import { env } from "@/lib/env"; import { isDemoMode } from "@/lib/env"; -import { jsonError } from "@/lib/http"; +import { jsonError, PublicApiError } from "@/lib/http"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; const signedUrlTtlSeconds = 60 * 10; +const routeIdSchema = z.string().uuid(); export async function GET(_request: Request, { params }: { params: Promise<{ id: string }> }) { try { @@ -24,6 +26,8 @@ export async function GET(_request: Request, { params }: { params: Promise<{ id: }); } + if (!routeIdSchema.safeParse(id).success) throw new PublicApiError("Invalid document id."); + const supabase = createAdminClient(); const user = await requireAuthenticatedUser(_request, supabase); const { data: document, error } = await supabase diff --git a/src/app/api/eval-cases/route.ts b/src/app/api/eval-cases/route.ts index a2f4039a9..7d7b0b7a3 100644 --- a/src/app/api/eval-cases/route.ts +++ b/src/app/api/eval-cases/route.ts @@ -2,8 +2,9 @@ import { NextResponse } from "next/server"; import { z } from "zod"; import { normalizedClinicalSearchTokens } from "@/lib/clinical-search"; import { clinicalQueryModeSchema } from "@/lib/clinical-query-mode"; -import { isDemoMode } from "@/lib/env"; +import { env, isDemoMode } from "@/lib/env"; import { jsonError, PublicApiError } from "@/lib/http"; +import { queryPrivacyMetadata, queryTextForStorage } from "@/lib/query-privacy"; import { searchScopeFiltersSchema } from "@/lib/search-scope"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -88,7 +89,10 @@ export async function POST(request: Request) { .from("rag_query_misses") .insert({ owner_id: user.id, - query: parsed.data.query, + // Raw clinical queries are potential PHI: store the normalized form unless + // raw retention is explicitly enabled (RET-H4), matching every other + // rag_query_misses / rag_queries writer. + query: queryTextForStorage(parsed.data.query), normalized_query: normalizedQuery, query_class: parsed.data.queryClass ?? parsed.data.queryMode, top_files: sourceFiles, @@ -101,11 +105,14 @@ export async function POST(request: Request) { promoted_eval_case: true, promoted_at: new Date().toISOString(), metadata: { + ...queryPrivacyMetadata(parsed.data.query), interaction: "answer_eval_capture", rating, feedback_type: parsed.data.feedbackType ?? null, - note: parsed.data.note, - answer: parsed.data.answer, + // Verbatim clinical note/answer are the larger PHI surface; only retain + // them when raw retention is explicitly enabled (RET-H4). + note: env.RAG_PERSIST_RAW_QUERY_TEXT ? parsed.data.note : "", + answer: env.RAG_PERSIST_RAW_QUERY_TEXT ? parsed.data.answer : "", query_class: parsed.data.queryClass ?? null, query_mode: parsed.data.queryMode, filters: parsed.data.filters ?? {}, diff --git a/src/app/api/images/[id]/signed-url/route.ts b/src/app/api/images/[id]/signed-url/route.ts index 2eacd226d..3f1a6d254 100644 --- a/src/app/api/images/[id]/signed-url/route.ts +++ b/src/app/api/images/[id]/signed-url/route.ts @@ -1,14 +1,16 @@ import { NextResponse } from "next/server"; +import { z } from "zod"; import { getDemoImage } from "@/lib/demo-data"; import { env } from "@/lib/env"; import { isDemoMode } from "@/lib/env"; -import { jsonError } from "@/lib/http"; +import { jsonError, PublicApiError } from "@/lib/http"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; export const runtime = "nodejs"; const signedUrlTtlSeconds = 60 * 10; +const routeIdSchema = z.string().uuid(); export async function GET(_request: Request, { params }: { params: Promise<{ id: string }> }) { try { @@ -25,6 +27,8 @@ export async function GET(_request: Request, { params }: { params: Promise<{ id: }); } + if (!routeIdSchema.safeParse(id).success) throw new PublicApiError("Invalid image id."); + const supabase = createAdminClient(); const user = await requireAuthenticatedUser(_request, supabase); const { data: image, error } = await supabase diff --git a/src/app/globals.css b/src/app/globals.css index 6c6f42680..e4137b84d 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -14,6 +14,7 @@ * rounded-xl bottom sheets, dialogs, large shells */ @theme { + --radius-xs: 0.25rem; --radius-sm: 0.375rem; --radius-md: 0.5rem; --radius-lg: 0.625rem; @@ -152,10 +153,9 @@ --disabled: #94a3b8; --focus: #1aa99b; - /* Motion primitives (shared by both themes) */ - --duration-fast: 150ms; - --duration-base: 200ms; - --duration-slow: 250ms; + /* Motion primitives (shared by both themes). The duration values live with + the easing tokens lower in this block (120/180/240ms); they are not + redeclared here to avoid shadowed, dead definitions. */ --ease-out-soft: cubic-bezier(0.22, 1, 0.36, 1); --ease-spring: cubic-bezier(0.34, 1.3, 0.64, 1); @@ -178,11 +178,9 @@ --space-8: 2rem; --space-12: 3rem; --space-16: 4rem; - --radius-xs: 0.25rem; - --radius-sm: 0.375rem; - --radius-md: 0.5rem; - --radius-lg: 0.75rem; - --radius-xl: 1rem; + /* Radius tokens are the single source of truth in @theme above (they also + generate the rounded-* utilities); do not redefine them here or var() and + the utilities drift apart. */ --duration-fast: 120ms; --duration-base: 180ms; --duration-slow: 240ms; diff --git a/src/components/DashboardFloatingFab.tsx b/src/components/DashboardFloatingFab.tsx index c2d119ed8..98aa73a04 100644 --- a/src/components/DashboardFloatingFab.tsx +++ b/src/components/DashboardFloatingFab.tsx @@ -1,7 +1,7 @@ "use client"; import Link from "next/link"; -import { useCallback, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { ArrowUp, ClipboardCopy, ExternalLink, Plus, Search, X } from "lucide-react"; import { cn, floatingControl } from "@/components/ui-primitives"; @@ -52,6 +52,10 @@ export function DashboardFloatingFab() { setOpen(false); }, []); + // Clear any pending copy-notice timeout on unmount so it can't fire after the + // component is gone (leaked timer / stray setState). + useEffect(() => clearNotice, [clearNotice]); + return (
diff --git a/src/lib/evidence.ts b/src/lib/evidence.ts index b298a5b44..fe1711f43 100644 --- a/src/lib/evidence.ts +++ b/src/lib/evidence.ts @@ -297,14 +297,18 @@ export function selectBestSourceRecommendation( const directQuote = quoteCards.find((quote) => quote.chunk_id === best.id); const documentQuote = quoteCards.find((quote) => quote.document_id === best.document_id); const quote = directQuote?.quote ?? documentQuote?.quote; - const snippet = quote ?? normalizeText(best.content).slice(0, 260).trim(); + // Track truncation by the pre-slice length rather than guessing from the + // post-trim snippet length (=== 260 dropped the ellipsis when trim shortened a + // truncated slice, and appended a spurious "..." to a complete 260-char quote). + const fullContent = normalizeText(best.content).trim(); + const snippet = quote ?? (fullContent.length > 260 ? `${fullContent.slice(0, 257).trimEnd()}...` : fullContent); const citation = citationFromResult(best); return { ...citation, source_strength: best.source_strength ?? sourceStrengthForSimilarity(best.similarity), score: best.hybrid_score ?? best.similarity, - snippet: snippet.length === 260 ? `${snippet.slice(0, 257).trim()}...` : snippet, + snippet, quote, section_heading: best.section_heading, image_count: (best.images ?? []).filter((image) => isClinicalImageEvidence(image)).length, diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts index b92110242..73daac961 100644 --- a/src/lib/extractors/document.ts +++ b/src/lib/extractors/document.ts @@ -146,7 +146,24 @@ async function extractDocx(buffer: Buffer) { if (file.dir) continue; const bytes = await file.async("nodebuffer"); const ext = path.extname(name).toLowerCase() || ".png"; - const mimeType = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : ext === ".webp" ? "image/webp" : "image/png"; + // Map the actual media extension to its real MIME type. Previously every + // non-jpg/webp extension (including .emf/.wmf/.tiff/.bmp/.gif vector and + // legacy-raster figures common in clinical .docx) was mislabeled image/png, + // which was then written to storage/DB and sent to the vision API as PNG. + const docxImageMimeByExt: Record = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + ".tif": "image/tiff", + ".tiff": "image/tiff", + ".emf": "image/emf", + ".wmf": "image/wmf", + ".svg": "image/svg+xml", + }; + const mimeType = docxImageMimeByExt[ext] ?? "application/octet-stream"; const outputPath = path.join(tempRoot, `docx-image-${index}${ext}`); await writeFile(outputPath, bytes); images.push({ diff --git a/src/lib/rag-answer-text.ts b/src/lib/rag-answer-text.ts index 095272be0..f8ecb3b18 100644 --- a/src/lib/rag-answer-text.ts +++ b/src/lib/rag-answer-text.ts @@ -8,6 +8,12 @@ const likelyFragmentPhrases = /\b(?:answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)\b/i; const answerSectionArtifactPattern = /"?(answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)"?\s*:\s*/i; +// Mid-text truncation must only fire on a genuine leaked JSON key — i.e. a +// double-quoted key followed by a colon ("confidence":). A bare English word + +// colon ("...document the confidence: high...") is legitimate clinical prose and +// must NOT cause the rest of the answer/section to be sliced away. +const leakedJsonKeyPattern = + /"(answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|conflictsOrGaps|quoteCards?|source_chunk_ids|chunk_id)"\s*:/i; export function normalizeSectionText(value: string) { return value.trim().replace(/\s+/g, " "); @@ -88,11 +94,15 @@ export function sanitizeStructuredText( const normalized = normalizeSectionText(sourceTextForClinicalProse(value)); if (!normalized) return ""; + // A leaked key at the very start is stripped (lenient: it is clearly an + // artifact prefix). Mid-text, only truncate at a genuine quoted JSON key so we + // never cut real prose at an ordinary word like "confidence:" or "body:". + const leakedKeyIndex = normalized.search(leakedJsonKeyPattern); const trimmed = normalized.search(answerSectionArtifactPattern) === 0 ? normalized.replace(answerSectionArtifactPattern, "").trim() - : normalized.search(answerSectionArtifactPattern) > 0 - ? normalized.slice(0, normalized.search(answerSectionArtifactPattern)).trim() + : leakedKeyIndex > 0 + ? normalized.slice(0, leakedKeyIndex).trim() : normalized; const finalText = keepLeading ? trimmed : trimmed.trim(); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index f9a40acf1..96ad7b46f 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3483,8 +3483,12 @@ function buildExtractiveAnswer(args: { } } for (const quote of quoteCards) { - if (!citationIds.has(quote.chunk_id)) - citations.push(resultCitation(args.results.find((result) => result.id === quote.chunk_id)!)); + if (!citationIds.has(quote.chunk_id)) { + // Guard the lookup: a quote card whose chunk_id was filtered out of results + // would make find() return undefined and resultCitation(undefined) throw. + const source = args.results.find((result) => result.id === quote.chunk_id); + if (source) citations.push(resultCitation(source)); + } citationIds.add(quote.chunk_id); } @@ -3506,10 +3510,15 @@ function buildExtractiveAnswer(args: { sourceCount: args.results.length, }); + // When no concise source sentence could be extracted, buildNaturalExtractiveAnswer + // returns a sentinel non-answer with no citationChunkIds. Do not present that as a + // grounded, confidence-rated answer just because compactCitations seeded a citation. + const hasExtractedAnswer = naturalAnswer.citationChunkIds.length > 0; + return { answer: naturalAnswer.answer, - grounded: citations.length > 0, - confidence: deriveConfidence(args.results, citations.length), + grounded: hasExtractedAnswer && citations.length > 0, + confidence: hasExtractedAnswer ? deriveConfidence(args.results, citations.length) : "low", citations: citations.slice(0, 5), sources: args.results, modelUsed: null, diff --git a/src/lib/search-scope.ts b/src/lib/search-scope.ts index bce64b94b..275f534b4 100644 --- a/src/lib/search-scope.ts +++ b/src/lib/search-scope.ts @@ -160,6 +160,10 @@ export async function resolveSearchScope(args: { .from("documents") .select("id,metadata,import_batch_id") .eq("status", "indexed") + // Deterministic total order over the unique id column. Without it, separate + // LIMIT/OFFSET page queries have no stable order, so rows can be silently + // skipped or duplicated across pages and the resolved scope is incomplete. + .order("id", { ascending: true }) .range(offset, Math.min(offset + documentScopeQueryPageSize - 1, maxResolvedDocuments - 1)); if (args.ownerId) documentQuery = documentQuery.eq("owner_id", args.ownerId); if (explicitIds.length) documentQuery = documentQuery.in("id", explicitIds); diff --git a/tests/eval-cases-route.test.ts b/tests/eval-cases-route.test.ts index 1c99febc2..2719a0122 100644 --- a/tests/eval-cases-route.test.ts +++ b/tests/eval-cases-route.test.ts @@ -37,7 +37,7 @@ afterEach(() => { describe("/api/eval-cases", () => { it("captures a good answer as a promoted eval case and filters malformed chunk ids", async () => { const { client, insert } = createInsertMock(); - vi.doMock("@/lib/env", () => ({ isDemoMode: () => false })); + vi.doMock("@/lib/env", () => ({ isDemoMode: () => false, env: { RAG_PERSIST_RAW_QUERY_TEXT: false } })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => client })); vi.doMock("@/lib/supabase/auth", () => ({ AuthenticationError: class AuthenticationError extends Error {}, @@ -63,6 +63,9 @@ describe("/api/eval-cases", () => { expect(response.status).toBe(201); expect(payload).toMatchObject({ owner_id: userId, + // PHI control (RET-H4): with raw retention OFF the verbatim query is NOT + // stored — only the normalized form. + query: "what monitoring is needed for clozapine?", query_class: "table_threshold", miss_reason: "answer_good_eval", top_files: ["CG.MHSP.ClozapinePresAdminMonitor.pdf"], @@ -77,12 +80,16 @@ describe("/api/eval-cases", () => { query_class: "table_threshold", source_chunk_ids_rejected: 1, cited_chunk_ids_rejected: 1, + // Privacy metadata folded in; verbatim answer dropped when retention is off. + raw_query_retained: false, + query_hash: expect.any(String), + answer: "", }); }); it("captures a needs-fixing answer without requiring expected UUID fields", async () => { const { client, insert } = createInsertMock(); - vi.doMock("@/lib/env", () => ({ isDemoMode: () => false })); + vi.doMock("@/lib/env", () => ({ isDemoMode: () => false, env: { RAG_PERSIST_RAW_QUERY_TEXT: false } })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => client })); vi.doMock("@/lib/supabase/auth", () => ({ AuthenticationError: class AuthenticationError extends Error {}, @@ -114,7 +121,7 @@ describe("/api/eval-cases", () => { it("captures category-specific missed-answer feedback for eval promotion", async () => { const { client, insert } = createInsertMock(); - vi.doMock("@/lib/env", () => ({ isDemoMode: () => false })); + vi.doMock("@/lib/env", () => ({ isDemoMode: () => false, env: { RAG_PERSIST_RAW_QUERY_TEXT: false } })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => client })); vi.doMock("@/lib/supabase/auth", () => ({ AuthenticationError: class AuthenticationError extends Error {}, diff --git a/worker/main.ts b/worker/main.ts index 8b56bb50b..22a99a2c7 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -1507,6 +1507,9 @@ async function processJob(job: JobRow) { } } finally { await cleanupExtractedTemporaryPaths(extracted); + // The progress-throttle entry is only needed while this job is processing; + // drop it so the module-level map does not grow for the worker's lifetime. + progressUpdateState.delete(job.id); } }