From 901fae59eca2c00c99c3b4ae79a84642b405672a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:13:33 +0800 Subject: [PATCH] perf: overlap the query embedding with lexical retrieval; cache typeahead catalogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 phase 5 (eval-gated; user-approved provider runs). - Speculative embedding: on the cold hybrid path the embedding round trip started only after every lexical layer had run, making it fully additive. It now starts alongside the lexical RPCs — but is consumed ONLY when expandedQuery is still byte-identical at the embed site, because the lexical layers can enrich the query with candidate metadata before embedding. Enriched queries take the serial path exactly as before, so retrieval results cannot change by construction. Cost: one wasted cached/coalesced embedding call when a lexical fast path wins (embedding_skipped_rate is ~0.72 on the golden set; embeddings are fractions of a cent). Never speculates for lexical-only (typeahead) or source-only retrieval. - Per-owner typeahead catalog cache (60s TTL): signed-in typeahead re-read the medications/services/forms catalogs from Supabase on every debounced keystroke; the mapped records are now cached per owner while ranking still runs per keystroke, so ranked output within the TTL is identical. - OPENAI_FAST_ANSWER_TIMEOUT_MS: optional tighter deadline for the fast answer tier only, so a hung fast generation can fall back / escalate before the full 30s answer timeout. Ships UNSET (behavior unchanged) — to be tuned in production against the Server-Timing/rag_queries generation-latency signal rather than local noise. Deliberately NOT done: trimming the per-keystroke lexical enrichment layers (ranking-metadata / memory-boost / visual-evidence) — memory boosting affects typeahead ORDERING, a real behavior change with poor risk/benefit after round 1 already removed the per-keystroke embedding. Eval gate (live Supabase + OpenAI, user-approved): - eval:retrieval baseline vs post-change: BIT-IDENTICAL correctness — 36 cases, document/content recall@5 = 1, mrr@10 0.8148, content_mrr@10 0.9279, identical retrieval strategy/plan/layer counts, failed_cases 0 (p90 latency 10407 -> 9658 ms). - eval:quality sanity (44 cases): citation failures 0, numeric grounding failures 0, governance danger failures 0, grounded-supported 0.9667, P95 16.7s. One case (clozapine-fbc-acronym-threshold) failed its 20s latency threshold on the strong route with correct retrieval — consistent with local-box latency noise; no generation behavior changes by default in this diff. Also: tsc, eslint, prettier, targeted vitest green. Co-Authored-By: Claude Fable 5 --- src/lib/env.ts | 6 ++++++ src/lib/rag.ts | 31 +++++++++++++++++++++++++++++-- src/lib/universal-search.ts | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/lib/env.ts b/src/lib/env.ts index dccf68890..6c489b163 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -55,6 +55,12 @@ const envSchema = z.object({ // than fail-closed; strong reasoning effort is also query-class-capped to keep the // tail latency in budget (see strongReasoningEffortForQueryClass). OPENAI_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().default(30000), + // Optional tighter deadline for the FAST answer tier only (SLO <=10s): a hung + // fast generation otherwise burns the full OPENAI_ANSWER_TIMEOUT_MS before it + // falls back or escalates. Unset = fast tier keeps OPENAI_ANSWER_TIMEOUT_MS + // (behavior unchanged); tune in production against real generation-latency + // telemetry before tightening. The strong tier always keeps the full timeout. + OPENAI_FAST_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().optional(), OPENAI_MAX_RETRIES: z.coerce.number().int().nonnegative().default(2), OPENAI_GENERATION_MAX_RETRIES: z.coerce.number().int().nonnegative().default(0), OPENAI_PROMPT_CACHE_RETENTION: z.enum(["off", "in_memory", "24h"]).default("24h"), diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 0927c6455..4da69b4fd 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3421,6 +3421,22 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { const minSimilarity = args.minSimilarity ?? 0.15; let embeddingStartedAt = 0; + // Speculative embedding (round-2 phase 5): on the cold hybrid path the + // embedding round trip started only after every lexical layer had run, so it + // was fully additive. Kick it off here to overlap the lexical RPCs. The + // lexical layers may ENRICH expandedQuery with candidate metadata before the + // real embed site — the speculation is only consumed when the query is still + // byte-identical there, so retrieval results cannot change; an enriched query + // falls back to the serial embed exactly as before (the speculative call is + // then one wasted cached/coalesced embedding). Never speculates for + // lexical-only (typeahead) or source-only retrieval, which skip embedding. + const speculativeEmbeddingQuery = expandedQuery; + const speculativeEmbedding = + sourceOnlyRetrieval || args.lexicalOnly ? null : embedTextWithTelemetry(speculativeEmbeddingQuery); + // Defers rejection handling to the await site below (auto-degrade semantics + // unchanged); without this a fast-path return would leave an unhandled rejection. + if (speculativeEmbedding) void speculativeEmbedding.catch(() => undefined); + let textFastResults: SearchResult[] = []; const textRpcStartedAt = Date.now(); const textData = await searchTextChunkCandidates({ @@ -3672,7 +3688,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { embeddingStartedAt = Date.now(); let embeddingResult: Awaited> | null = null; try { - embeddingResult = await embedTextWithTelemetry(expandedQuery); + // embedding_latency_ms now measures the residual wait at this point: near + // zero when the speculative call overlapped the lexical layers, the full + // round trip when metadata enrichment changed the query (serial fallback). + embeddingResult = + speculativeEmbedding && expandedQuery === speculativeEmbeddingQuery + ? await speculativeEmbedding + : await embedTextWithTelemetry(expandedQuery); } catch (error) { // In auto mode a failed embedding call (e.g. quota exhausted) degrades to the lexical // results already gathered rather than failing the whole search. "openai" mode rethrows. @@ -4696,7 +4718,12 @@ ${qualityRetryInstruction}` schemaName: "clinical_rag_answer", instructions: answerInstructions, promptCacheKey: "clinical-rag-answer-v18", - timeoutMs: env.OPENAI_ANSWER_TIMEOUT_MS, + // The fast tier may carry a tighter deadline (OPENAI_FAST_ANSWER_TIMEOUT_MS) + // so a hung generation falls back / escalates before the full answer + // timeout; unset it falls back to the shared timeout (no behavior change). + timeoutMs: useStrongReasoning + ? env.OPENAI_ANSWER_TIMEOUT_MS + : (env.OPENAI_FAST_ANSWER_TIMEOUT_MS ?? env.OPENAI_ANSWER_TIMEOUT_MS), maxRetries: 0, reasoningEffort: useStrongReasoning ? strongReasoningEffortForQueryClass(queryClass, env.OPENAI_STRONG_REASONING_EFFORT) diff --git a/src/lib/universal-search.ts b/src/lib/universal-search.ts index 2a598efc6..ea48fa9f9 100644 --- a/src/lib/universal-search.ts +++ b/src/lib/universal-search.ts @@ -170,10 +170,35 @@ function formItem(record: FormRecord, score: number): UniversalSearchItem { }; } +// Per-owner registry catalogs are small seeded sets that change rarely, but +// signed-in typeahead re-read all three (medications/services/forms) from +// Supabase on every debounced keystroke. Cache the mapped records per owner for +// a short TTL — ranking still runs per keystroke, so within the TTL the ranked +// output is identical to an uncached read. Single-process, like the module's +// alias cache. +const ownerCatalogTtlMs = 60_000; +const ownerCatalogCacheMax = 128; +const ownerCatalogCache = new Map(); + +async function cachedOwnerCatalog(cacheKey: string, load: () => Promise): Promise { + const cached = ownerCatalogCache.get(cacheKey); + if (cached && cached.expiresAtMs > Date.now()) return cached.records as T[]; + const records = await load(); + ownerCatalogCache.set(cacheKey, { records, expiresAtMs: Date.now() + ownerCatalogTtlMs }); + while (ownerCatalogCache.size > ownerCatalogCacheMax) { + const oldestKey = ownerCatalogCache.keys().next().value; + if (!oldestKey) break; + ownerCatalogCache.delete(oldestKey); + } + return records; +} + async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerMedicationRowsWithSeed(args.supabase, args.ownerId)).map(rowToMedicationRecord) + ? await cachedOwnerCatalog(`medications:${args.ownerId}`, async () => + (await fetchOwnerMedicationRowsWithSeed(args.supabase!, args.ownerId!)).map(rowToMedicationRecord), + ) : defaultMedicationRecords(); return rankMedicationRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => medicationItem(match.medication, match.score), @@ -183,7 +208,9 @@ async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "service")).map(rowToServiceRecord) + ? await cachedOwnerCatalog(`services:${args.ownerId}`, async () => + (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "service")).map(rowToServiceRecord), + ) : serviceRecords; return rankServiceRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => serviceItem(match.service, match.score), @@ -193,7 +220,9 @@ async function searchServicesDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "form")).map(rowToServiceRecord) + ? await cachedOwnerCatalog(`forms:${args.ownerId}`, async () => + (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "form")).map(rowToServiceRecord), + ) : formRecords; return rankFormRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => formItem(match.service, match.score),