fix(ci): unblock deployment boot smoke and pace eval canary

.github/workflows/eval-canary.yml

@@ -38,6 +38,13 @@ env:
   NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key
   SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  # Pace golden retrieval so forced-vector probes do not trip transient 429s
+  # after a long text-fast-path run (see vector-ptsd history in eval-canary).
+  RAG_EVAL_CASE_DELAY_MS: 2000
+  RAG_EVAL_FORCE_EMBEDDING_DELAY_MS: 5000
+  RAG_EVAL_PROVIDER_RETRY_ATTEMPTS: 6
+  RAG_EVAL_PROVIDER_RETRY_INITIAL_MS: 8000
+  RAG_EVAL_PROVIDER_RETRY_MAX_MS: 120000
   # The live corpus is all-public (owner_id = NULL). Owner-scoped retrieval
   # fail-closes on a null owner, so the eval must run as the public-owner
   # sentinel. resolveEvalOwnerId (scripts/eval-utils.ts) defaults to that

docs/observability-slos.md

@@ -157,6 +157,10 @@ Operational notes:
   failure can be corpus-state-dependent (see the clozapine-wcc history).
   Triage order: rerun via `workflow_dispatch` → check `hybrid_rpc_errors` and
   `check:indexing` → only then bisect code.
+- Forced-vector golden cases (`forceEmbedding`) run after many text-fast-path
+  cases; the workflow sets `RAG_EVAL_CASE_DELAY_MS` and
+  `RAG_EVAL_FORCE_EMBEDDING_DELAY_MS` so embedding calls do not exhaust the
+  nightly OpenAI rate limit mid-run.
 - Evals write telemetry rows (`rag_queries`) but mutate no content.
 - Cost bound: ~34 retrieval cases (embedding calls only on forced-vector
   probes) + 8 generated answers per night.

scripts/eval-retrieval.ts

@@ -5,7 +5,13 @@ import { loadEnvConfig } from "@next/env";
 import { z } from "zod";
 import { loadCapturedRagEvalCases, type RagEvalCase, type SupabaseEvalCaseClient } from "@/lib/rag-eval-cases";
 import type { SearchResult } from "@/lib/types";
-import { loadAdminClient, percentile, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils";
+import {
+  loadAdminClient,
+  pauseBetweenEvalCases,
+  percentile,
+  resolveEvalOwnerId,
+  withProviderBackoff,
+} from "./eval-utils";
 
 loadEnvConfig(process.cwd());
 
@@ -830,7 +836,12 @@ async function main() {
     for (const warning of readinessWarnings) console.warn(`WARN ${warning}`);
   }
 
-  for (const testCase of cases) {
+  for (let caseIndex = 0; caseIndex < cases.length; caseIndex += 1) {
+    const testCase = cases[caseIndex]!;
+    await pauseBetweenEvalCases({
+      caseIndex,
+      forceEmbedding: testCase.forceEmbedding || args.forceEmbedding,
+    });
     const startedAt = Date.now();
     const searchPromise = withProviderBackoff(`retrieval:${testCase.id}`, () =>
       searchChunksWithTelemetry({

scripts/eval-utils.ts

@@ -15,6 +15,23 @@ function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+export function evalCaseDelayMs() {
+  const parsed = Number.parseInt(process.env.RAG_EVAL_CASE_DELAY_MS ?? "", 10);
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0;
+}
+
+export function evalForceEmbeddingDelayMs() {
+  const parsed = Number.parseInt(process.env.RAG_EVAL_FORCE_EMBEDDING_DELAY_MS ?? "", 10);
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0;
+}
+
+export async function pauseBetweenEvalCases(options: { caseIndex: number; forceEmbedding?: boolean }) {
+  if (options.caseIndex <= 0) return;
+  let delayMs = evalCaseDelayMs();
+  if (options.forceEmbedding) delayMs += evalForceEmbeddingDelayMs();
+  if (delayMs > 0) await sleep(delayMs);
+}
+
 function providerRetryNumber(value: string | undefined, fallback: number) {
   const parsed = Number.parseInt(value ?? "", 10);
   return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;

tests/eval-utils.test.ts

@@ -3,6 +3,7 @@ import {
   DEFAULT_EVAL_OWNER_ID,
   expectedFileCoverage,
   isProviderRateLimitError,
+  pauseBetweenEvalCases,
   resolveEvalOwnerId,
   validateRagAnswer,
   withProviderBackoff,
@@ -97,6 +98,24 @@ describe("RAG eval source identity matching", () => {
     expect(attempts).toBe(2);
     expect(isProviderRateLimitError(new Error("429 too many requests"))).toBe(true);
   });
+
+  it("pauses between eval cases when configured", async () => {
+    vi.useFakeTimers();
+    try {
+      vi.stubEnv("RAG_EVAL_CASE_DELAY_MS", "15");
+      vi.stubEnv("RAG_EVAL_FORCE_EMBEDDING_DELAY_MS", "15");
+
+      const pausePromise = pauseBetweenEvalCases({ caseIndex: 1, forceEmbedding: true });
+      await vi.advanceTimersByTimeAsync(30);
+      await pausePromise;
+
+      await pauseBetweenEvalCases({ caseIndex: 0, forceEmbedding: true });
+      expect(vi.getTimerCount()).toBe(0);
+    } finally {
+      vi.useRealTimers();
+      vi.unstubAllEnvs();
+    }
+  });
 });
 
 describe("resolveEvalOwnerId", () => {

tests/ui-universal-search.spec.ts

@@ -49,7 +49,7 @@ async function mockUniversalSearch(page: Page) {
 }
 
 async function openComposer(page: Page) {
-  await page.goto("/?mode=documents&focus=1");
+  await page.goto("/?mode=documents&focus=1", { waitUntil: "domcontentloaded" });
   const input = page.getByTestId("global-search-input").first();
   await input.click();
   return input;