perf: overlap query embedding with lexical retrieval + typeahead catalog cache (round 2, phase 5)

src/lib/env.ts

@@ -55,6 +55,12 @@ const envSchema = z.object({
   // than fail-closed; strong reasoning effort is also query-class-capped to keep the
   // tail latency in budget (see strongReasoningEffortForQueryClass).
   OPENAI_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().default(30000),
+  // Optional tighter deadline for the FAST answer tier only (SLO <=10s): a hung
+  // fast generation otherwise burns the full OPENAI_ANSWER_TIMEOUT_MS before it
+  // falls back or escalates. Unset = fast tier keeps OPENAI_ANSWER_TIMEOUT_MS
+  // (behavior unchanged); tune in production against real generation-latency
+  // telemetry before tightening. The strong tier always keeps the full timeout.
+  OPENAI_FAST_ANSWER_TIMEOUT_MS: z.coerce.number().int().positive().optional(),
   OPENAI_MAX_RETRIES: z.coerce.number().int().nonnegative().default(2),
   OPENAI_GENERATION_MAX_RETRIES: z.coerce.number().int().nonnegative().default(0),
   OPENAI_PROMPT_CACHE_RETENTION: z.enum(["off", "in_memory", "24h"]).default("24h"),

src/lib/rag.ts

@@ -3421,6 +3421,22 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
   const minSimilarity = args.minSimilarity ?? 0.15;
   let embeddingStartedAt = 0;
 
+  // Speculative embedding (round-2 phase 5): on the cold hybrid path the
+  // embedding round trip started only after every lexical layer had run, so it
+  // was fully additive. Kick it off here to overlap the lexical RPCs. The
+  // lexical layers may ENRICH expandedQuery with candidate metadata before the
+  // real embed site — the speculation is only consumed when the query is still
+  // byte-identical there, so retrieval results cannot change; an enriched query
+  // falls back to the serial embed exactly as before (the speculative call is
+  // then one wasted cached/coalesced embedding). Never speculates for
+  // lexical-only (typeahead) or source-only retrieval, which skip embedding.
+  const speculativeEmbeddingQuery = expandedQuery;
+  const speculativeEmbedding =
+    sourceOnlyRetrieval || args.lexicalOnly ? null : embedTextWithTelemetry(speculativeEmbeddingQuery);
+  // Defers rejection handling to the await site below (auto-degrade semantics
+  // unchanged); without this a fast-path return would leave an unhandled rejection.
+  if (speculativeEmbedding) void speculativeEmbedding.catch(() => undefined);
+
   let textFastResults: SearchResult[] = [];
   const textRpcStartedAt = Date.now();
   const textData = await searchTextChunkCandidates({
@@ -3672,7 +3688,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) {
   embeddingStartedAt = Date.now();
   let embeddingResult: Awaited<ReturnType<typeof embedTextWithTelemetry>> | null = null;
   try {
-    embeddingResult = await embedTextWithTelemetry(expandedQuery);
+    // embedding_latency_ms now measures the residual wait at this point: near
+    // zero when the speculative call overlapped the lexical layers, the full
+    // round trip when metadata enrichment changed the query (serial fallback).
+    embeddingResult =
+      speculativeEmbedding && expandedQuery === speculativeEmbeddingQuery
+        ? await speculativeEmbedding
+        : await embedTextWithTelemetry(expandedQuery);
   } catch (error) {
     // In auto mode a failed embedding call (e.g. quota exhausted) degrades to the lexical
     // results already gathered rather than failing the whole search. "openai" mode rethrows.
@@ -4696,7 +4718,12 @@ ${qualityRetryInstruction}`
         schemaName: "clinical_rag_answer",
         instructions: answerInstructions,
         promptCacheKey: "clinical-rag-answer-v18",
-        timeoutMs: env.OPENAI_ANSWER_TIMEOUT_MS,
+        // The fast tier may carry a tighter deadline (OPENAI_FAST_ANSWER_TIMEOUT_MS)
+        // so a hung generation falls back / escalates before the full answer
+        // timeout; unset it falls back to the shared timeout (no behavior change).
+        timeoutMs: useStrongReasoning
+          ? env.OPENAI_ANSWER_TIMEOUT_MS
+          : (env.OPENAI_FAST_ANSWER_TIMEOUT_MS ?? env.OPENAI_ANSWER_TIMEOUT_MS),
         maxRetries: 0,
         reasoningEffort: useStrongReasoning
           ? strongReasoningEffortForQueryClass(queryClass, env.OPENAI_STRONG_REASONING_EFFORT)

src/lib/universal-search.ts

@@ -170,10 +170,35 @@ function formItem(record: FormRecord, score: number): UniversalSearchItem {
   };
 }
 
+// Per-owner registry catalogs are small seeded sets that change rarely, but
+// signed-in typeahead re-read all three (medications/services/forms) from
+// Supabase on every debounced keystroke. Cache the mapped records per owner for
+// a short TTL — ranking still runs per keystroke, so within the TTL the ranked
+// output is identical to an uncached read. Single-process, like the module's
+// alias cache.
+const ownerCatalogTtlMs = 60_000;
+const ownerCatalogCacheMax = 128;
+const ownerCatalogCache = new Map<string, { records: unknown[]; expiresAtMs: number }>();
+
+async function cachedOwnerCatalog<T>(cacheKey: string, load: () => Promise<T[]>): Promise<T[]> {
+  const cached = ownerCatalogCache.get(cacheKey);
+  if (cached && cached.expiresAtMs > Date.now()) return cached.records as T[];
+  const records = await load();
+  ownerCatalogCache.set(cacheKey, { records, expiresAtMs: Date.now() + ownerCatalogTtlMs });
+  while (ownerCatalogCache.size > ownerCatalogCacheMax) {
+    const oldestKey = ownerCatalogCache.keys().next().value;
+    if (!oldestKey) break;
+    ownerCatalogCache.delete(oldestKey);
+  }
+  return records;
+}
+
 async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise<UniversalSearchItem[]> {
   const records =
     !args.demo && args.supabase && args.ownerId
-      ? (await fetchOwnerMedicationRowsWithSeed(args.supabase, args.ownerId)).map(rowToMedicationRecord)
+      ? await cachedOwnerCatalog(`medications:${args.ownerId}`, async () =>
+          (await fetchOwnerMedicationRowsWithSeed(args.supabase!, args.ownerId!)).map(rowToMedicationRecord),
+        )
       : defaultMedicationRecords();
   return rankMedicationRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) =>
     medicationItem(match.medication, match.score),
@@ -183,7 +208,9 @@ async function searchMedicationsDomain(args: ResolvedSearchArgs): Promise<Univer
 async function searchServicesDomain(args: ResolvedSearchArgs): Promise<UniversalSearchItem[]> {
   const records =
     !args.demo && args.supabase && args.ownerId
-      ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "service")).map(rowToServiceRecord)
+      ? await cachedOwnerCatalog(`services:${args.ownerId}`, async () =>
+          (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "service")).map(rowToServiceRecord),
+        )
       : serviceRecords;
   return rankServiceRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) =>
     serviceItem(match.service, match.score),
@@ -193,7 +220,9 @@ async function searchServicesDomain(args: ResolvedSearchArgs): Promise<Universal
 async function searchFormsDomain(args: ResolvedSearchArgs): Promise<UniversalSearchItem[]> {
   const records =
     !args.demo && args.supabase && args.ownerId
-      ? (await fetchOwnerRegistryRowsWithSeed(args.supabase, args.ownerId, "form")).map(rowToServiceRecord)
+      ? await cachedOwnerCatalog(`forms:${args.ownerId}`, async () =>
+          (await fetchOwnerRegistryRowsWithSeed(args.supabase!, args.ownerId!, "form")).map(rowToServiceRecord),
+        )
       : formRecords;
   return rankFormRecords(records, args.baseQuery, args.limitPerDomain, args.expansions).map((match) =>
     formItem(match.service, match.score),