diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index 0697554b3..0c2afd5e0 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -38,6 +38,13 @@ env: NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # Pace golden retrieval so forced-vector probes do not trip transient 429s + # after a long text-fast-path run (see vector-ptsd history in eval-canary). + RAG_EVAL_CASE_DELAY_MS: 2000 + RAG_EVAL_FORCE_EMBEDDING_DELAY_MS: 5000 + RAG_EVAL_PROVIDER_RETRY_ATTEMPTS: 6 + RAG_EVAL_PROVIDER_RETRY_INITIAL_MS: 8000 + RAG_EVAL_PROVIDER_RETRY_MAX_MS: 120000 # The live corpus is all-public (owner_id = NULL). Owner-scoped retrieval # fail-closes on a null owner, so the eval must run as the public-owner # sentinel. resolveEvalOwnerId (scripts/eval-utils.ts) defaults to that diff --git a/docs/observability-slos.md b/docs/observability-slos.md index 519458bd7..2dc48c74a 100644 --- a/docs/observability-slos.md +++ b/docs/observability-slos.md @@ -157,6 +157,10 @@ Operational notes: failure can be corpus-state-dependent (see the clozapine-wcc history). Triage order: rerun via `workflow_dispatch` → check `hybrid_rpc_errors` and `check:indexing` → only then bisect code. +- Forced-vector golden cases (`forceEmbedding`) run after many text-fast-path + cases; the workflow sets `RAG_EVAL_CASE_DELAY_MS` and + `RAG_EVAL_FORCE_EMBEDDING_DELAY_MS` so embedding calls do not exhaust the + nightly OpenAI rate limit mid-run. - Evals write telemetry rows (`rag_queries`) but mutate no content. - Cost bound: ~34 retrieval cases (embedding calls only on forced-vector probes) + 8 generated answers per night. diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 1e3ec67f0..6645fef3c 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -5,7 +5,13 @@ import { loadEnvConfig } from "@next/env"; import { z } from "zod"; import { loadCapturedRagEvalCases, type RagEvalCase, type SupabaseEvalCaseClient } from "@/lib/rag-eval-cases"; import type { SearchResult } from "@/lib/types"; -import { loadAdminClient, percentile, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils"; +import { + loadAdminClient, + pauseBetweenEvalCases, + percentile, + resolveEvalOwnerId, + withProviderBackoff, +} from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -830,7 +836,12 @@ async function main() { for (const warning of readinessWarnings) console.warn(`WARN ${warning}`); } - for (const testCase of cases) { + for (let caseIndex = 0; caseIndex < cases.length; caseIndex += 1) { + const testCase = cases[caseIndex]!; + await pauseBetweenEvalCases({ + caseIndex, + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, + }); const startedAt = Date.now(); const searchPromise = withProviderBackoff(`retrieval:${testCase.id}`, () => searchChunksWithTelemetry({ diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index a4cc0ebce..66ddb3e81 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -15,6 +15,23 @@ function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +export function evalCaseDelayMs() { + const parsed = Number.parseInt(process.env.RAG_EVAL_CASE_DELAY_MS ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +export function evalForceEmbeddingDelayMs() { + const parsed = Number.parseInt(process.env.RAG_EVAL_FORCE_EMBEDDING_DELAY_MS ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +export async function pauseBetweenEvalCases(options: { caseIndex: number; forceEmbedding?: boolean }) { + if (options.caseIndex <= 0) return; + let delayMs = evalCaseDelayMs(); + if (options.forceEmbedding) delayMs += evalForceEmbeddingDelayMs(); + if (delayMs > 0) await sleep(delayMs); +} + function providerRetryNumber(value: string | undefined, fallback: number) { const parsed = Number.parseInt(value ?? "", 10); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; diff --git a/tests/eval-utils.test.ts b/tests/eval-utils.test.ts index 961f54e4b..0c952d0d4 100644 --- a/tests/eval-utils.test.ts +++ b/tests/eval-utils.test.ts @@ -3,6 +3,7 @@ import { DEFAULT_EVAL_OWNER_ID, expectedFileCoverage, isProviderRateLimitError, + pauseBetweenEvalCases, resolveEvalOwnerId, validateRagAnswer, withProviderBackoff, @@ -97,6 +98,24 @@ describe("RAG eval source identity matching", () => { expect(attempts).toBe(2); expect(isProviderRateLimitError(new Error("429 too many requests"))).toBe(true); }); + + it("pauses between eval cases when configured", async () => { + vi.useFakeTimers(); + try { + vi.stubEnv("RAG_EVAL_CASE_DELAY_MS", "15"); + vi.stubEnv("RAG_EVAL_FORCE_EMBEDDING_DELAY_MS", "15"); + + const pausePromise = pauseBetweenEvalCases({ caseIndex: 1, forceEmbedding: true }); + await vi.advanceTimersByTimeAsync(30); + await pausePromise; + + await pauseBetweenEvalCases({ caseIndex: 0, forceEmbedding: true }); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + } + }); }); describe("resolveEvalOwnerId", () => { diff --git a/tests/ui-universal-search.spec.ts b/tests/ui-universal-search.spec.ts index 4ce4909ef..e40f82ec7 100644 --- a/tests/ui-universal-search.spec.ts +++ b/tests/ui-universal-search.spec.ts @@ -49,7 +49,7 @@ async function mockUniversalSearch(page: Page) { } async function openComposer(page: Page) { - await page.goto("/?mode=documents&focus=1"); + await page.goto("/?mode=documents&focus=1", { waitUntil: "domcontentloaded" }); const input = page.getByTestId("global-search-input").first(); await input.click(); return input;