diff --git a/scripts/reindex-image-generation-metadata.ts b/scripts/reindex-image-generation-metadata.ts index 3d0f4904e..ced585e07 100644 --- a/scripts/reindex-image-generation-metadata.ts +++ b/scripts/reindex-image-generation-metadata.ts @@ -80,19 +80,14 @@ function parseArgs(argv: string[]): Args { continue; } + if (token !== "--document-id" && token !== "--owner-id" && token !== "--limit") continue; + const value = argv[index + 1]; - if (!value || value.startsWith("--")) { - if (token === "--document-id" || token === "--owner-id" || token === "--limit") { - throw new Error(`Missing value for ${token}`); - } - continue; - } + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); if (token === "--document-id") args.documentId = value; - if (token === "--owner-id") args.ownerId = value; - if (token === "--limit") args.limit = Number.parseInt(value, 10); - if (token === "--document-id" || token === "--owner-id" || token === "--limit") { - index += 1; - } + else if (token === "--owner-id") args.ownerId = value; + else args.limit = Number.parseInt(value, 10); + index += 1; } if (!Number.isFinite(args.limit) || args.limit <= 0) { @@ -105,11 +100,8 @@ function parseArgs(argv: string[]): Args { } function rowNeedsRefresh(row: ImageRow, committedGeneration: string) { - const rowMetadataGeneration = committedIndexGeneration(row.metadata); if (row.index_generation_id !== null && row.index_generation_id !== committedGeneration) return true; - return row.index_generation_id === null - ? rowMetadataGeneration !== committedGeneration - : rowMetadataGeneration !== committedGeneration; + return committedIndexGeneration(row.metadata) !== committedGeneration; } async function loadDocuments(args: { diff --git a/scripts/seed-differential-records.ts b/scripts/seed-differential-records.ts index 0283258c9..2120bd5a1 100644 --- a/scripts/seed-differential-records.ts +++ b/scripts/seed-differential-records.ts @@ -3,7 +3,7 @@ import { loadEnvConfig } from "@next/env"; import { confirm } from "./cli-utils"; import { buildDefaultDifferentialRows, loadDifferentialSnapshot } from "@/lib/differential-fixtures"; import type { DifferentialRecordRow } from "@/lib/differential-records"; -import { embedDifferentialRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { embedDifferentialRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; loadEnvConfig(process.cwd()); @@ -126,13 +126,12 @@ async function main() { } if (registryCorpusEmbeddingEnabled()) { - const { data: embeddedRows, error: embedRowsError } = await supabase - .from("differential_records") - .select("*") - .eq("owner_id", args.ownerId); - if (embedRowsError) throw new Error(`Could not reload differential rows for embedding: ${embedRowsError.message}`); - const embedded = await embedDifferentialRows(supabase, (embeddedRows ?? []) as DifferentialRecordRow[]); - console.log(`[differentials:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`); + const chunkCount = await embedReloadedOwnerRows( + supabase.from("differential_records").select("*").eq("owner_id", args.ownerId), + (rows) => embedDifferentialRows(supabase, rows as DifferentialRecordRow[]), + "differential", + ); + console.log(`[differentials:seed] Embedded ${chunkCount} registry corpus chunk(s).`); } } diff --git a/scripts/seed-medication-records.ts b/scripts/seed-medication-records.ts index 09582e05a..0806ac820 100644 --- a/scripts/seed-medication-records.ts +++ b/scripts/seed-medication-records.ts @@ -2,7 +2,7 @@ import { loadEnvConfig } from "@next/env"; import { confirm } from "./cli-utils"; import { buildMedicationSeedRows } from "@/lib/medication-seed"; -import { embedMedicationRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { embedMedicationRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; import type { MedicationRecordRow } from "@/lib/medication-records"; loadEnvConfig(process.cwd()); @@ -123,13 +123,12 @@ async function main() { } if (registryCorpusEmbeddingEnabled()) { - const { data: embeddedRows, error: embedRowsError } = await supabase - .from("medication_records") - .select("*") - .eq("owner_id", args.ownerId); - if (embedRowsError) throw new Error(`Could not reload medication rows for embedding: ${embedRowsError.message}`); - const embedded = await embedMedicationRows(supabase, (embeddedRows ?? []) as MedicationRecordRow[]); - console.log(`[medications:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`); + const chunkCount = await embedReloadedOwnerRows( + supabase.from("medication_records").select("*").eq("owner_id", args.ownerId), + (rows) => embedMedicationRows(supabase, rows as MedicationRecordRow[]), + "medication", + ); + console.log(`[medications:seed] Embedded ${chunkCount} registry corpus chunk(s).`); } } diff --git a/scripts/seed-registry-records.ts b/scripts/seed-registry-records.ts index 97bc0482b..5d548b89c 100644 --- a/scripts/seed-registry-records.ts +++ b/scripts/seed-registry-records.ts @@ -1,7 +1,7 @@ import { loadEnvConfig } from "@next/env"; import { confirm } from "./cli-utils"; -import { embedClinicalRegistryRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { embedClinicalRegistryRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; import type { RegistryRecordRow } from "@/lib/registry-records"; import { type RegistryRecordKind } from "@/lib/registry-records"; import { buildDefaultRegistryRows, defaultRegistryRecords } from "@/lib/registry-seed"; @@ -149,14 +149,12 @@ async function main() { if (registryCorpusEmbeddingEnabled()) { const kinds: RegistryRecordKind[] = args.kind === "all" ? ["service", "form"] : [args.kind]; - const { data: embeddedRows, error: embedRowsError } = await supabase - .from("clinical_registry_records") - .select("*") - .eq("owner_id", args.ownerId) - .in("kind", kinds); - if (embedRowsError) throw new Error(`Could not reload registry rows for embedding: ${embedRowsError.message}`); - const embedded = await embedClinicalRegistryRows(supabase, (embeddedRows ?? []) as RegistryRecordRow[]); - console.log(`[registry:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`); + const chunkCount = await embedReloadedOwnerRows( + supabase.from("clinical_registry_records").select("*").eq("owner_id", args.ownerId).in("kind", kinds), + (rows) => embedClinicalRegistryRows(supabase, rows as RegistryRecordRow[]), + "registry", + ); + console.log(`[registry:seed] Embedded ${chunkCount} registry corpus chunk(s).`); } } diff --git a/src/lib/differential-seed.ts b/src/lib/differential-seed.ts index c1b17d3f2..4e36421e4 100644 --- a/src/lib/differential-seed.ts +++ b/src/lib/differential-seed.ts @@ -1,6 +1,6 @@ import { buildDefaultDifferentialRows } from "@/lib/differential-fixtures"; import { type DifferentialRecordInsert, type DifferentialRecordRow } from "@/lib/differential-records"; -import { embedDifferentialRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { bestEffortEmbedRows, embedDifferentialRows } from "@/lib/registry-corpus"; type AdminClient = ReturnType; @@ -17,13 +17,11 @@ export async function ensureDifferentialsSeeded( .select("*"); if (error) throw new Error(`Differential seed failed: ${error.message}`); const seededRows = (data ?? []) as DifferentialRecordRow[]; - if (registryCorpusEmbeddingEnabled()) { - try { - await embedDifferentialRows(supabase, seededRows); - } catch (embedError) { - console.error(`[differentials] corpus embedding failed for owner ${ownerId}`, embedError); - } - } + await bestEffortEmbedRows({ + scope: "differentials", + ownerId, + embed: () => embedDifferentialRows(supabase, seededRows), + }); return seededRows; } diff --git a/src/lib/medication-seed.ts b/src/lib/medication-seed.ts index 78179fb41..e02850e37 100644 --- a/src/lib/medication-seed.ts +++ b/src/lib/medication-seed.ts @@ -1,5 +1,5 @@ import { buildDefaultMedicationRows, defaultMedicationRecords } from "@/lib/medication-fixtures"; -import { embedMedicationRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { bestEffortEmbedRows, embedMedicationRows } from "@/lib/registry-corpus"; import { type MedicationRecordInsert, type MedicationRecordRow } from "@/lib/medication-records"; type AdminClient = ReturnType; @@ -16,13 +16,11 @@ export async function ensureMedicationsSeeded(supabase: AdminClient, ownerId: st .select("*"); if (error) throw new Error(`Medication seed failed: ${error.message}`); const seededRows = (data ?? []) as MedicationRecordRow[]; - if (registryCorpusEmbeddingEnabled()) { - try { - await embedMedicationRows(supabase, seededRows); - } catch (embedError) { - console.error(`[medications] corpus embedding failed for owner ${ownerId}`, embedError); - } - } + await bestEffortEmbedRows({ + scope: "medications", + ownerId, + embed: () => embedMedicationRows(supabase, seededRows), + }); return seededRows; } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index b33e6fb58..11c45fe9f 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -2376,9 +2376,8 @@ async function attachDocumentRankingMetadata( for (const row of metadataRows) { cache.documentMetadata.set(row.document_id, { labels: row.labels, summary: row.summary }); } - const metadataByDocument = new Map(metadataRows.map((row) => [row.document_id, row])); const enriched = results.map((result) => { - const metadata = metadataByDocument.get(result.document_id) ?? cache.documentMetadata.get(result.document_id); + const metadata = cache.documentMetadata.get(result.document_id); if (!metadata) return result; return { ...result, @@ -2392,6 +2391,13 @@ async function attachDocumentRankingMetadata( } } +function withCachedIndexQuality(results: SearchResult[], cache: DocumentRankingMetadataCache) { + return results.map((result) => ({ + ...result, + indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null, + })); +} + async function attachIndexQualityMetadata( supabase: ReturnType, results: SearchResult[], @@ -2401,12 +2407,7 @@ async function attachIndexQualityMetadata( const documentIds = Array.from(new Set(results.map((result) => result.document_id))); if (documentIds.length === 0) return results; const missingDocumentIds = documentIds.filter((documentId) => !cache.indexQuality.has(documentId)); - if (missingDocumentIds.length === 0) { - return results.map((result) => ({ - ...result, - indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null, - })); - } + if (missingDocumentIds.length === 0) return withCachedIndexQuality(results, cache); try { let query = supabase .from("document_index_quality") @@ -2417,15 +2418,7 @@ async function attachIndexQualityMetadata( if (error) return results; for (const documentId of missingDocumentIds) cache.indexQuality.set(documentId, null); for (const row of data ?? []) cache.indexQuality.set(row.document_id, row as SearchResult["indexing_quality"]); - const qualityByDocument = new Map((data ?? []).map((row) => [row.document_id, row])); - return results.map((result) => ({ - ...result, - indexing_quality: - (qualityByDocument.get(result.document_id) as SearchResult["indexing_quality"]) ?? - cache.indexQuality.get(result.document_id) ?? - result.indexing_quality ?? - null, - })); + return withCachedIndexQuality(results, cache); } catch { return results; } diff --git a/src/lib/registry-corpus.ts b/src/lib/registry-corpus.ts index 8a3bcfc1c..854270a1a 100644 --- a/src/lib/registry-corpus.ts +++ b/src/lib/registry-corpus.ts @@ -80,11 +80,17 @@ function registryBaseMetadata(entry: RegistryCorpusEntry): Record }; } +function registryDocumentId(entry: RegistryCorpusEntry) { + return deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`); +} + +function registryEntryMetadata(entry: RegistryCorpusEntry): Record { + return { ...registryBaseMetadata(entry), ...entry.metadata }; +} + function registryDocumentRow(entry: RegistryCorpusEntry): TablesInsert<"documents"> { - const documentId = deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`); - const metadata = { ...registryBaseMetadata(entry), ...entry.metadata }; return { - id: documentId, + id: registryDocumentId(entry), owner_id: entry.ownerId, title: entry.title, description: entry.subtitle, @@ -98,16 +104,14 @@ function registryDocumentRow(entry: RegistryCorpusEntry): TablesInsert<"document chunk_count: 1, image_count: 0, content_hash: sha256(entry.content), - metadata, + metadata: registryEntryMetadata(entry), }; } function registryChunkRow(entry: RegistryCorpusEntry, embedding: Vector): TablesInsert<"document_chunks"> { - const documentId = deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`); - const metadata = { ...registryBaseMetadata(entry), ...entry.metadata }; return { id: deterministicUuid(`registry-chunk:${entry.kind}:${entry.recordId}`), - document_id: documentId, + document_id: registryDocumentId(entry), chunk_index: 0, page_number: 1, section_heading: "Registry summary", @@ -118,7 +122,7 @@ function registryChunkRow(entry: RegistryCorpusEntry, embedding: Vector): Tables image_ids: [], content_hash: sha256(entry.content), embedding, - metadata, + metadata: registryEntryMetadata(entry), }; } @@ -275,3 +279,30 @@ export function embedMedicationRows(supabase: AdminClient, rows: MedicationRecor export function embedDifferentialRows(supabase: AdminClient, rows: DifferentialRecordRow[]) { return embedRegistryCorpusEntries(supabase, differentialRowsToCorpusEntries(rows)); } + +/** Best-effort corpus embedding for a seed path: failures are logged, not thrown, + * so a broken embedding call never blocks the seed write it runs after. */ +export async function bestEffortEmbedRows(args: { scope: string; ownerId: string; detail?: string; embed: () => Promise }) { + if (!registryCorpusEmbeddingEnabled()) return; + try { + await args.embed(); + } catch (embedError) { + const suffix = args.detail ? ` ${args.detail}` : ""; + console.error(`[${args.scope}] corpus embedding failed for owner ${args.ownerId}${suffix}`, embedError); + } +} + +/** Reload an owner's rows for a table and embed them, returning the chunk count + * written. Shared by the registry/medication/differential seed CLIs, which each + * re-read their own rows (rather than reusing the upsert response) so the + * embedded corpus always reflects what is actually stored. */ +export async function embedReloadedOwnerRows( + reload: PromiseLike<{ data: Row[] | null; error: { message: string } | null }>, + embed: (rows: Row[]) => Promise<{ chunkCount: number }>, + tableLabel: string, +) { + const { data, error } = await reload; + if (error) throw new Error(`Could not reload ${tableLabel} rows for embedding: ${error.message}`); + const { chunkCount } = await embed(data ?? []); + return chunkCount; +} diff --git a/src/lib/registry-seed.ts b/src/lib/registry-seed.ts index 4078b3879..0b42f24e6 100644 --- a/src/lib/registry-seed.ts +++ b/src/lib/registry-seed.ts @@ -1,6 +1,6 @@ import { formRecords } from "@/lib/forms"; import { buildDefaultFormRows, buildDefaultServiceRows, defaultServiceRecords } from "@/lib/registry-fixtures"; -import { embedClinicalRegistryRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus"; +import { bestEffortEmbedRows, embedClinicalRegistryRows } from "@/lib/registry-corpus"; import { type RegistryRecordInsert, type RegistryRecordKind, type RegistryRecordRow } from "@/lib/registry-records"; // Type-only reference to the admin client so this module carries no runtime @@ -44,13 +44,12 @@ export async function ensureRegistrySeeded( .select("*"); if (error) throw new Error(`Registry seed failed: ${error.message}`); const seededRows = (data ?? []) as RegistryRecordRow[]; - if (registryCorpusEmbeddingEnabled()) { - try { - await embedClinicalRegistryRows(supabase, seededRows); - } catch (embedError) { - console.error(`[registry] corpus embedding failed for owner ${ownerId} (${kind})`, embedError); - } - } + await bestEffortEmbedRows({ + scope: "registry", + ownerId, + detail: `(${kind})`, + embed: () => embedClinicalRegistryRows(supabase, seededRows), + }); return seededRows; }