diff --git a/docs/operator-backlog.md b/docs/operator-backlog.md index ad000700..eb749675 100644 --- a/docs/operator-backlog.md +++ b/docs/operator-backlog.md @@ -17,15 +17,15 @@ Findings inventory for handover: [audit-handover-2026-07-14.md](audit-handover-2 ## Launch-gating actions -| Action | Status | Blocked by | Verify command | Runbook | -| -------------------------------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Apply July-8 migration batch (a–g) to live | ✅ done | — | `SUPABASE_ENVIRONMENT=production npm run check:july8-live-batch` (2026-07-13: 6 live, apply=no-op) | [operator-apply-july8-batch.md](operator-apply-july8-batch.md) | -| Apply drift-codify forward migration (step 1h) | ✅ done | — | Applied and drift/readiness verified 2026-07-13; verify only unless new reviewed drift is found | [database-drift-detection.md](database-drift-detection.md) | -| Apply repo-ahead migrations to live (post-2026-07-13) | ⏳ pending | Queue includes `20260717120000`, `20260717170000`, and `20260717171000`; a forward migration to purge private/non-indexed `document_title_words` rows must be merged and reviewed before apply | Zero unsafe title-word rows; `npm run check:drift`; then `eval:retrieval:quality` (36/36) for the corrector | [deploy-corrector-public-titles.md](deploy-corrector-public-titles.md) · [operator-apply-performance-latency-remediation.md](operator-apply-performance-latency-remediation.md) | -| Full release gate (bounded OpenAI spend) | ⏳ pending | migrations 1 applied | `npm run verify:release`; `npm run eval:quality -- --rag-only` | [launch-operator-runbook.md §2](launch-operator-runbook.md) | -| Provision staging Supabase project (`Clinical KB Staging`, ap-southeast-2) | ⏳ pending | — | `npm run check:indexing` after `db push` | [staging-setup.md](staging-setup.md) | -| Staging soak + rollback rehearsal on Railway | ⏳ pending | staging provisioned | `scripts/soak-test.ts --confirm-staging` (answer p95 ≤ 25 s) | [launch-operator-runbook.md §4](launch-operator-runbook.md) · [capacity-review.md](capacity-review.md) | -| Production deploy to Railway | ✅ done | — | App deployment recorded live 2026-07-14; re-verify with `GET /api/health` and deployment readiness | [deployment-architecture.md](deployment-architecture.md) | +| Action | Status | Blocked by | Verify command | Runbook | +| -------------------------------------------------------------------------- | ---------- | -------------------- | ----------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Apply July-8 migration batch (a–g) to live | ✅ done | — | `SUPABASE_ENVIRONMENT=production npm run check:july8-live-batch` (2026-07-13: 6 live, apply=no-op) | [operator-apply-july8-batch.md](operator-apply-july8-batch.md) | +| Apply drift-codify forward migration (step 1h) | ✅ done | — | Applied and drift/readiness verified 2026-07-13; verify only unless new reviewed drift is found | [database-drift-detection.md](database-drift-detection.md) | +| Apply repo-ahead migrations to live (post-2026-07-13) | ✅ done | — | Zero unsafe title-word rows; `npm run check:drift`; then `eval:retrieval:quality` (36/36) for the corrector | [deploy-corrector-public-titles.md](deploy-corrector-public-titles.md) · [operator-apply-performance-latency-remediation.md](operator-apply-performance-latency-remediation.md) | +| Full release gate (bounded OpenAI spend) | ⏳ pending | migrations 1 applied | `npm run verify:release`; `npm run eval:quality -- --rag-only` | [launch-operator-runbook.md §2](launch-operator-runbook.md) | +| Provision staging Supabase project (`Clinical KB Staging`, ap-southeast-2) | ⏳ pending | — | `npm run check:indexing` after `db push` | [staging-setup.md](staging-setup.md) | +| Staging soak + rollback rehearsal on Railway | ⏳ pending | staging provisioned | `scripts/soak-test.ts --confirm-staging` (answer p95 ≤ 25 s) | [launch-operator-runbook.md §4](launch-operator-runbook.md) · [capacity-review.md](capacity-review.md) | +| Production deploy to Railway | ✅ done | — | App deployment recorded live 2026-07-14; re-verify with `GET /api/health` and deployment readiness | [deployment-architecture.md](deployment-architecture.md) | ## Post-deploy actions diff --git a/src/app/api/documents/[id]/signed-url/route.ts b/src/app/api/documents/[id]/signed-url/route.ts index 6a42543b..8f045d6a 100644 --- a/src/app/api/documents/[id]/signed-url/route.ts +++ b/src/app/api/documents/[id]/signed-url/route.ts @@ -53,10 +53,10 @@ export async function GET(_request: Request, { params }: { params: Promise<{ id: const registryHref = (typeof metadata.registry_detail_href === "string" && metadata.registry_detail_href) || registryCorpusDetailHref({ - kind: metadata.registry_record_kind, - slug: metadata.registry_record_slug, - subkind: metadata.registry_record_subkind, - recordId: metadata.registry_record_id, + kind: metadata.registry_record_kind as string | undefined, + slug: metadata.registry_record_slug as string | undefined, + subkind: metadata.registry_record_subkind as string | undefined, + recordId: metadata.registry_record_id as string | undefined, }); if (source.source_kind === "registry_record" && registryHref) { return NextResponse.json({ diff --git a/src/components/clinical-dashboard/source-actions.tsx b/src/components/clinical-dashboard/source-actions.tsx index fe26992a..4be5a98a 100644 --- a/src/components/clinical-dashboard/source-actions.tsx +++ b/src/components/clinical-dashboard/source-actions.tsx @@ -67,10 +67,10 @@ export function sourceResultHref(source: SearchResult) { ? (source.source_metadata as Record) : {}; const registryHref = registryCorpusDetailHref({ - kind: metadata.registry_record_kind, - slug: metadata.registry_record_slug, - subkind: metadata.registry_record_subkind, - recordId: metadata.registry_record_id, + kind: metadata.registry_record_kind as string | undefined, + slug: metadata.registry_record_slug as string | undefined, + subkind: metadata.registry_record_subkind as string | undefined, + recordId: metadata.registry_record_id as string | undefined, }); if (registryHref) return registryHref; return `/documents/${source.document_id}?page=${source.page_number ?? 1}&chunk=${source.id}`; diff --git a/src/lib/api-rate-limit.ts b/src/lib/api-rate-limit.ts index d87f5d8f..58130ead 100644 --- a/src/lib/api-rate-limit.ts +++ b/src/lib/api-rate-limit.ts @@ -349,10 +349,17 @@ function consumeInMemoryApiRateLimit({ const windowMs = windowSeconds * 1000; const key = `${ownerId}:${bucket}`; - // Evict expired entries to prevent memory leak - for (const [k, v] of inMemoryApiRateLimits.entries()) { - if (now >= v.resetAtMs) { - inMemoryApiRateLimits.delete(k); + // Lazy per-key eviction: the most common path touches only the accessed entry. + // A full-map sweep runs only when the map exceeds a size ceiling so stale entries + // don't accumulate indefinitely, without scanning on every request. + const EVICTION_SIZE_CEILING = 2000; + const stale = inMemoryApiRateLimits.get(key); + if (stale && now >= stale.resetAtMs) { + inMemoryApiRateLimits.delete(key); + } + if (inMemoryApiRateLimits.size > EVICTION_SIZE_CEILING) { + for (const [k, v] of inMemoryApiRateLimits.entries()) { + if (now >= v.resetAtMs) inMemoryApiRateLimits.delete(k); } } diff --git a/src/lib/citations.ts b/src/lib/citations.ts index 2a277667..4098feb4 100644 --- a/src/lib/citations.ts +++ b/src/lib/citations.ts @@ -86,9 +86,9 @@ function registryCitationHref(citation: Citation) { if (!slug) return null; return registryCorpusDetailHref({ - kind: metadata.registry_record_kind, + kind: metadata.registry_record_kind ?? undefined, slug: encodeURIComponent(slug), - subkind: metadata.registry_record_subkind, + subkind: metadata.registry_record_subkind ?? undefined, }); } diff --git a/src/lib/env.ts b/src/lib/env.ts index e1b2681e..6b319296 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -68,6 +68,7 @@ const envSchema = z.object({ // batches of this size. 256 keeps total tokens well under the ceiling even for the // largest (narrative-profile) chunks while staying far below the 2048 input cap. OPENAI_EMBEDDING_BATCH_SIZE: z.coerce.number().int().positive().max(2048).default(256), + OPENAI_EMBEDDING_CONCURRENCY_LIMIT: z.coerce.number().int().positive().default(5), OPENAI_VISION_MODEL: z.string().default("gpt-5.6-terra"), OPENAI_VISION_IMAGE_DETAIL: z.enum(["auto", "low", "high"]).default("auto"), OPENAI_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(45000), diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 47d54412..d9b444c2 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -365,7 +365,11 @@ function isTimeoutError(error: unknown) { } export function mapOpenAIError(error: unknown, operation: OpenAIOperation) { - if (error instanceof PublicApiError) return error; + if ( + error instanceof PublicApiError || + (error && typeof error === "object" && "name" in error && error.name === "PublicApiError") + ) + return error as PublicApiError; const status = getErrorStatus(error); const code = getErrorCode(error) ?? "openai_request_failed"; @@ -580,48 +584,84 @@ export async function embedTexts(texts: string[], options?: { signal?: AbortSign // embedding of an unrelated text, even across batch boundaries (extends IDX-C1). const byIndex = new Array(uniqueTexts.length); const batches = chunkIntoBatches(uniqueTexts, env.OPENAI_EMBEDDING_BATCH_SIZE); - let batchStart = 0; - for (const batch of batches) { - const response = await client.embeddings.create( - { - model: env.OPENAI_EMBEDDING_MODEL, - input: batch, - // IDX-C2: request the exact dimension the schema's vector(N) columns expect. - dimensions: env.EMBEDDING_DIMENSIONS, - }, - requestOptions({ operation: "embedding", signal: options?.signal }), - ); - - // IDX-C2: a short response means some inputs silently produced no embedding. - if (response.data.length !== batch.length) { - throw new PublicApiError( - `OpenAI returned ${response.data.length} embeddings for ${batch.length} inputs.`, - 502, - { code: "openai_embedding_count_mismatch" }, - ); - } - // IDX-C1: the embeddings API does not guarantee response order; each item carries - // an explicit `index` into the request's input array. Reassemble by that index - // (offset to the global position) so embeddings are never mismatched to chunks. - for (const item of response.data) { - if (item.index < 0 || item.index >= batch.length) { - throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { - code: "openai_embedding_index_range", - }); - } - // IDX-C2: guard against a model whose dimension does not match the schema. - if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { - throw new PublicApiError( - `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + - `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, - 502, - { code: "openai_embedding_dimension_mismatch" }, + // IDX-C3: a single embeddings request is capped at 2048 inputs / ~300k tokens, so a + // full-corpus re-embed must be split. To maximize throughput while respecting rate + // limits, batches run concurrently up to env.OPENAI_EMBEDDING_CONCURRENCY_LIMIT (default: 5). + // Reassembly uses the GLOBAL index (task.offset + item.index) so a chunk is never stored with the + // embedding of an unrelated text, even across batch/concurrency boundaries (extends IDX-C1). + let currentOffset = 0; + const tasks = batches.map((batch) => { + const offset = currentOffset; + currentOffset += batch.length; + return { batch, offset }; + }); + + const concurrencyLimit = env.OPENAI_EMBEDDING_CONCURRENCY_LIMIT; + let nextTaskIndex = 0; + let firstError: unknown = null; + + const worker = async () => { + while (nextTaskIndex < tasks.length && !firstError) { + if (options?.signal?.aborted) break; + + const task = tasks[nextTaskIndex++]; + if (!task) break; + + try { + const response = await client.embeddings.create( + { + model: env.OPENAI_EMBEDDING_MODEL, + input: task.batch, + // IDX-C2: request the exact dimension the schema's vector(N) columns expect. + dimensions: env.EMBEDDING_DIMENSIONS, + }, + requestOptions({ operation: "embedding", signal: options?.signal }), ); + + // IDX-C2: a short response means some inputs silently produced no embedding. + if (response.data.length !== task.batch.length) { + throw new PublicApiError( + `OpenAI returned ${response.data.length} embeddings for ${task.batch.length} inputs.`, + 502, + { code: "openai_embedding_count_mismatch" }, + ); + } + + // IDX-C1: the embeddings API does not guarantee response order; each item carries + // an explicit `index` into the request's input array. Reassemble by that index + // (offset to the global position) so embeddings are never mismatched to chunks. + for (const item of response.data) { + if (item.index < 0 || item.index >= task.batch.length) { + throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { + code: "openai_embedding_index_range", + }); + } + // IDX-C2: guard against a model whose dimension does not match the schema. + if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { + throw new PublicApiError( + `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + + `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, + 502, + { code: "openai_embedding_dimension_mismatch" }, + ); + } + byIndex[task.offset + item.index] = item.embedding; + } + } catch (err) { + if (!firstError) { + firstError = err; + } + break; } - byIndex[batchStart + item.index] = item.embedding; } - batchStart += batch.length; + }; + + const workers = Array.from({ length: Math.min(concurrencyLimit, tasks.length) }, worker); + await Promise.all(workers); + + if (firstError) { + throw firstError; } return outputIndexes.map((index) => byIndex[index]); diff --git a/src/lib/registry-corpus-links.ts b/src/lib/registry-corpus-links.ts index 57bc47d1..c305f259 100644 --- a/src/lib/registry-corpus-links.ts +++ b/src/lib/registry-corpus-links.ts @@ -1,8 +1,10 @@ +import type { RegistryCorpusKind } from "./registry-corpus"; + export function registryCorpusDetailHref(args: { - kind: unknown; - slug: unknown; - subkind?: unknown; - recordId?: unknown; + kind: RegistryCorpusKind | string | null | undefined; + slug: string | null | undefined; + subkind?: string | null | undefined; + recordId?: string | null | undefined; }) { const kind = typeof args.kind === "string" ? args.kind : null; const slug = typeof args.slug === "string" ? args.slug : null; diff --git a/src/lib/universal-search.ts b/src/lib/universal-search.ts index 8b957f73..a2fd6b72 100644 --- a/src/lib/universal-search.ts +++ b/src/lib/universal-search.ts @@ -426,10 +426,10 @@ function searchResultDocumentHref(result: SearchResult) { ? (result.source_metadata as Record) : {}; const registryHref = registryCorpusDetailHref({ - kind: metadata.registry_record_kind, - slug: metadata.registry_record_slug, - subkind: metadata.registry_record_subkind, - recordId: metadata.registry_record_id, + kind: metadata.registry_record_kind as string | undefined, + slug: metadata.registry_record_slug as string | undefined, + subkind: metadata.registry_record_subkind as string | undefined, + recordId: metadata.registry_record_id as string | undefined, }); if (registryHref) return registryHref; return `/documents/${result.document_id}`; diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index 6b783cf9..c1812754 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-07-19T10:21:34.307Z", + "generated_at": "2026-07-20T08:31:04.870Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127", - "schema_sha256": "f4b18b6fc86646a43daba7b73ef0c61a52ede0ac43bfeef5d73a56779a238871", - "replay_seconds": 75, + "schema_sha256": "50da0978a164058f2bfe539a77c011902b52f91f6c006c67ed0ff7c700b464c5", + "replay_seconds": 16, "snapshot": { "views": [ { @@ -5430,6 +5430,12 @@ "table": "documents", "def_hash": "d516bc49b5f81bb31779e01d3d52c11b" }, + { + "def": "CREATE INDEX documents_owner_updated_at_indexed_idx ON public.documents USING btree (owner_id, updated_at DESC) WHERE (status = 'indexed'::text)", + "name": "documents_owner_updated_at_indexed_idx", + "table": "documents", + "def_hash": "d821f4b02d3d865d6764c174dbc5ca3c" + }, { "def": "CREATE UNIQUE INDEX documents_pkey ON public.documents USING btree (id)", "name": "documents_pkey", diff --git a/supabase/migrations/20260720170000_add_documents_owner_updated_at_indexed_idx.sql b/supabase/migrations/20260720170000_add_documents_owner_updated_at_indexed_idx.sql new file mode 100644 index 00000000..b24755c5 --- /dev/null +++ b/supabase/migrations/20260720170000_add_documents_owner_updated_at_indexed_idx.sql @@ -0,0 +1,4 @@ +-- Add a partial composite index to optimize cacheIndexingVersion scans +create index if not exists documents_owner_updated_at_indexed_idx + on public.documents (owner_id, updated_at desc) + where status = 'indexed'; diff --git a/supabase/schema.sql b/supabase/schema.sql index 02d214c5..768d1f4a 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -6200,6 +6200,7 @@ create index if not exists document_summaries_owner_id_idx on public.document_su create index if not exists document_table_facts_document_generation_idx on public.document_table_facts (document_id, index_generation_id) where index_generation_id is not null; create index if not exists document_table_facts_title_row_param_trgm_idx on public.document_table_facts using gin (lower(coalesce(table_title, '') || ' ' || coalesce(row_label, '') || ' ' || coalesce(clinical_parameter, '')) extensions.gin_trgm_ops); create index if not exists documents_indexed_updated_at_idx on public.documents (updated_at, id) where status = 'indexed'; +create index if not exists documents_owner_updated_at_indexed_idx on public.documents (owner_id, updated_at desc) where status = 'indexed'; create index if not exists image_caption_cache_owner_id_idx on public.image_caption_cache (owner_id); create index if not exists import_batches_owner_id_idx on public.import_batches (owner_id); create index if not exists ingestion_job_stages_job_idx on public.ingestion_job_stages (job_id, started_at desc); diff --git a/tests/registry-corpus.test.ts b/tests/registry-corpus.test.ts index 2c1c5b7d..b988c8d8 100644 --- a/tests/registry-corpus.test.ts +++ b/tests/registry-corpus.test.ts @@ -395,12 +395,12 @@ describe("registry corpus", () => { expect( registryCorpusDetailHref({ - kind: metadata.registry_record_kind, - slug: metadata.registry_record_slug, - subkind: metadata.registry_record_subkind, - recordId: metadata.registry_record_id, + kind: metadata.registry_record_kind as string | undefined, + slug: metadata.registry_record_slug as string | undefined, + subkind: metadata.registry_record_subkind as string | undefined, + recordId: metadata.registry_record_id as string | undefined, }), ).toBe("/forms/adult-adhd-screen"); - expect(registryCorpusDetailHref({ kind: "service", slug: 42 })).toBeNull(); + expect(registryCorpusDetailHref({ kind: "service", slug: 42 as unknown as string })).toBeNull(); }); }); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 537c9751..50918824 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -1217,7 +1217,7 @@ describe("Supabase Preview replay guards", () => { const migrationFiles = readdirSync(migrationDirectoryUrl) .filter((fileName) => /^\d+_.+\.sql$/.test(fileName)) .sort(); - expect(migrationFiles).toContain("20260719053533_enforce_public_title_word_scope.sql"); + expect(migrationFiles).toContain("20260720170000_add_documents_owner_updated_at_indexed_idx.sql"); expect(documentTitleWordScopeMigration).toContain( "v_status := public.default_privileges_status('postgres', 'public')", );