diff --git a/src/lib/env.ts b/src/lib/env.ts index dccf68890..6c489b163 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -55,6 +55,12 @@ const envSchema = z.object({ // than fail-closed; strong reasoning effort is also query-class-capped to keep the // tail latency in budget (see strongReasoningEffortForQueryClass). OPENAI_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().default(30000), + // Optional tighter deadline for the FAST answer tier only (SLO <=10s): a hung + // fast generation otherwise burns the full OPENAI_ANSWER_TIMEOUT_MS before it + // falls back or escalates. Unset = fast tier keeps OPENAI_ANSWER_TIMEOUT_MS + // (behavior unchanged); tune in production against real generation-latency + // telemetry before tightening. The strong tier always keeps the full timeout. + OPENAI_FAST_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().optional(), OPENAI_MAX_RETRIES: z.coerce.number().int().nonnegative().default(2), OPENAI_GENERATION_MAX_RETRIES: z.coerce.number().int().nonnegative().default(0), OPENAI_PROMPT_CACHE_RETENTION: z.enum(["off", "in_memory", "24h"]).default("24h"), diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 0927c6455..4da69b4fd 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3421,6 +3421,22 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { const minSimilarity = args.minSimilarity ?? 0.15; let embeddingStartedAt = 0; + // Speculative embedding (round-2 phase 5): on the cold hybrid path the + // embedding round trip started only after every lexical layer had run, so it + // was fully additive. Kick it off here to overlap the lexical RPCs. The + // lexical layers may ENRICH expandedQuery with candidate metadata before the + // real embed site — the speculation is only consumed when the query is still + // byte-identical there, so retrieval results cannot change; an enriched query + // falls back to the serial embed exactly as before (the speculative call is + // then one wasted cached/coalesced embedding). Never speculates for + // lexical-only (typeahead) or source-only retrieval, which skip embedding. + const speculativeEmbeddingQuery = expandedQuery; + const speculativeEmbedding = + sourceOnlyRetrieval || args.lexicalOnly ? null : embedTextWithTelemetry(speculativeEmbeddingQuery); + // Defers rejection handling to the await site below (auto-degrade semantics + // unchanged); without this a fast-path return would leave an unhandled rejection. + if (speculativeEmbedding) void speculativeEmbedding.catch(() => undefined); + let textFastResults: SearchResult[] = []; const textRpcStartedAt = Date.now(); const textData = await searchTextChunkCandidates({ @@ -3672,7 +3688,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { embeddingStartedAt = Date.now(); let embeddingResult: Awaited> | null = null; try { - embeddingResult = await embedTextWithTelemetry(expandedQuery); + // embedding_latency_ms now measures the residual wait at this point: near + // zero when the speculative call overlapped the lexical layers, the full + // round trip when metadata enrichment changed the query (serial fallback). + embeddingResult = + speculativeEmbedding && expandedQuery === speculativeEmbeddingQuery + ? await speculativeEmbedding + : await embedTextWithTelemetry(expandedQuery); } catch (error) { // In auto mode a failed embedding call (e.g. quota exhausted) degrades to the lexical // results already gathered rather than failing the whole search. "openai" mode rethrows. @@ -4696,7 +4718,12 @@ ${qualityRetryInstruction}` schemaName: "clinical_rag_answer", instructions: answerInstructions, promptCacheKey: "clinical-rag-answer-v18", - timeoutMs: env.OPENAI_ANSWER_TIMEOUT_MS, + // The fast tier may carry a tighter deadline (OPENAI_FAST_ANSWER_TIMEOUT_MS) + // so a hung generation falls back / escalates before the full answer + // timeout; unset it falls back to the shared timeout (no behavior change). + timeoutMs: useStrongReasoning + ? env.OPENAI_ANSWER_TIMEOUT_MS + : (env.OPENAI_FAST_ANSWER_TIMEOUT_MS ?? env.OPENAI_ANSWER_TIMEOUT_MS), maxRetries: 0, reasoningEffort: useStrongReasoning ? strongReasoningEffortForQueryClass(queryClass, env.OPENAI_STRONG_REASONING_EFFORT) diff --git a/src/lib/universal-search.ts b/src/lib/universal-search.ts index 2a598efc6..ea48fa9f9 100644 --- a/src/lib/universal-search.ts +++ b/src/lib/universal-search.ts @@ -170,10 +170,35 @@ function formItem(record: FormRecord, score: number): UniversalSearchItem { }; } +// Per-owner registry catalogs are small seeded sets that change rarely, but +// signed-in typeahead re-read all three (medications/services/forms) from +// Supabase on every debounced keystroke. Cache the mapped records per owner for +// a short TTL — ranking still runs per keystroke, so within the TTL the ranked +// output is identical to an uncached read. Single-process, like the module's +// alias cache. +const ownerCatalogTtlMs = 60_000; +const ownerCatalogCacheMax = 128; +const ownerCatalogCache = new Map(); + +async function cachedOwnerCatalog(cacheKey: string, load: () => Promise): Promise { + const cached = ownerCatalogCache.get(cacheKey); + if (cached && cached.expiresAtMs > Date.now()) return cached.records as T[]; + const records = await load(); + ownerCatalogCache.set(cacheKey, { records, expiresAtMs: Date.now() + ownerCatalogTtlMs }); + while (ownerCatalogCache.size > ownerCatalogCacheMax) { + const oldestKey = ownerCatalogCache.keys().next().value; + if (!oldestKey) break; + ownerCatalogCache.delete(oldestKey); + } + return records; +} + async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerMedicationRowsWithSeed(args.supabase, args.ownerId)).map(rowToMedicationRecord) + ? await cachedOwnerCatalog(`medications:${args.ownerId}`, async () => + (await fetchOwnerMedicationRowsWithSeed(args.supabase!, args.ownerId!)).map(rowToMedicationRecord), + ) : defaultMedicationRecords(); return rankMedicationRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => medicationItem(match.medication, match.score), @@ -183,7 +208,9 @@ async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "service")).map(rowToServiceRecord) + ? await cachedOwnerCatalog(`services:${args.ownerId}`, async () => + (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "service")).map(rowToServiceRecord), + ) : serviceRecords; return rankServiceRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => serviceItem(match.service, match.score), @@ -193,7 +220,9 @@ async function searchServicesDomain(args: ResolvedSearchArgs): Promise { const records = !args.demo && args.supabase && args.ownerId - ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "form")).map(rowToServiceRecord) + ? await cachedOwnerCatalog(`forms:${args.ownerId}`, async () => + (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "form")).map(rowToServiceRecord), + ) : formRecords; return rankFormRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) => formItem(match.service, match.score),