Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions scripts/reindex-image-generation-metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,19 +80,14 @@ function parseArgs(argv: string[]): Args {
continue;
}

if (token !== "--document-id" && token !== "--owner-id" && token !== "--limit") continue;

const value = argv[index + 1];
if (!value || value.startsWith("--")) {
if (token === "--document-id" || token === "--owner-id" || token === "--limit") {
throw new Error(`Missing value for ${token}`);
}
continue;
}
if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
if (token === "--document-id") args.documentId = value;
if (token === "--owner-id") args.ownerId = value;
if (token === "--limit") args.limit = Number.parseInt(value, 10);
if (token === "--document-id" || token === "--owner-id" || token === "--limit") {
index += 1;
}
else if (token === "--owner-id") args.ownerId = value;
else args.limit = Number.parseInt(value, 10);
index += 1;
}

if (!Number.isFinite(args.limit) || args.limit <= 0) {
Expand All @@ -105,11 +100,8 @@ function parseArgs(argv: string[]): Args {
}

function rowNeedsRefresh(row: ImageRow, committedGeneration: string) {
const rowMetadataGeneration = committedIndexGeneration(row.metadata);
if (row.index_generation_id !== null && row.index_generation_id !== committedGeneration) return true;
return row.index_generation_id === null
? rowMetadataGeneration !== committedGeneration
: rowMetadataGeneration !== committedGeneration;
return committedIndexGeneration(row.metadata) !== committedGeneration;
}

async function loadDocuments(args: {
Expand Down
15 changes: 7 additions & 8 deletions scripts/seed-differential-records.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { loadEnvConfig } from "@next/env";
import { confirm } from "./cli-utils";
import { buildDefaultDifferentialRows, loadDifferentialSnapshot } from "@/lib/differential-fixtures";
import type { DifferentialRecordRow } from "@/lib/differential-records";
import { embedDifferentialRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { embedDifferentialRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";

loadEnvConfig(process.cwd());

Expand Down Expand Up @@ -126,13 +126,12 @@ async function main() {
}

if (registryCorpusEmbeddingEnabled()) {
const { data: embeddedRows, error: embedRowsError } = await supabase
.from("differential_records")
.select("*")
.eq("owner_id", args.ownerId);
if (embedRowsError) throw new Error(`Could not reload differential rows for embedding: ${embedRowsError.message}`);
const embedded = await embedDifferentialRows(supabase, (embeddedRows ?? []) as DifferentialRecordRow[]);
console.log(`[differentials:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`);
const chunkCount = await embedReloadedOwnerRows(
supabase.from("differential_records").select("*").eq("owner_id", args.ownerId),
(rows) => embedDifferentialRows(supabase, rows as DifferentialRecordRow[]),
"differential",
);
console.log(`[differentials:seed] Embedded ${chunkCount} registry corpus chunk(s).`);
}
}

Expand Down
15 changes: 7 additions & 8 deletions scripts/seed-medication-records.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { loadEnvConfig } from "@next/env";

import { confirm } from "./cli-utils";
import { buildMedicationSeedRows } from "@/lib/medication-seed";
import { embedMedicationRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { embedMedicationRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import type { MedicationRecordRow } from "@/lib/medication-records";

loadEnvConfig(process.cwd());
Expand Down Expand Up @@ -123,13 +123,12 @@ async function main() {
}

if (registryCorpusEmbeddingEnabled()) {
const { data: embeddedRows, error: embedRowsError } = await supabase
.from("medication_records")
.select("*")
.eq("owner_id", args.ownerId);
if (embedRowsError) throw new Error(`Could not reload medication rows for embedding: ${embedRowsError.message}`);
const embedded = await embedMedicationRows(supabase, (embeddedRows ?? []) as MedicationRecordRow[]);
console.log(`[medications:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`);
const chunkCount = await embedReloadedOwnerRows(
supabase.from("medication_records").select("*").eq("owner_id", args.ownerId),
(rows) => embedMedicationRows(supabase, rows as MedicationRecordRow[]),
"medication",
);
console.log(`[medications:seed] Embedded ${chunkCount} registry corpus chunk(s).`);
}
}

Expand Down
16 changes: 7 additions & 9 deletions scripts/seed-registry-records.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { loadEnvConfig } from "@next/env";

import { confirm } from "./cli-utils";
import { embedClinicalRegistryRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { embedClinicalRegistryRows, embedReloadedOwnerRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import type { RegistryRecordRow } from "@/lib/registry-records";
import { type RegistryRecordKind } from "@/lib/registry-records";
import { buildDefaultRegistryRows, defaultRegistryRecords } from "@/lib/registry-seed";
Expand Down Expand Up @@ -149,14 +149,12 @@ async function main() {

if (registryCorpusEmbeddingEnabled()) {
const kinds: RegistryRecordKind[] = args.kind === "all" ? ["service", "form"] : [args.kind];
const { data: embeddedRows, error: embedRowsError } = await supabase
.from("clinical_registry_records")
.select("*")
.eq("owner_id", args.ownerId)
.in("kind", kinds);
if (embedRowsError) throw new Error(`Could not reload registry rows for embedding: ${embedRowsError.message}`);
const embedded = await embedClinicalRegistryRows(supabase, (embeddedRows ?? []) as RegistryRecordRow[]);
console.log(`[registry:seed] Embedded ${embedded.chunkCount} registry corpus chunk(s).`);
const chunkCount = await embedReloadedOwnerRows(
supabase.from("clinical_registry_records").select("*").eq("owner_id", args.ownerId).in("kind", kinds),
(rows) => embedClinicalRegistryRows(supabase, rows as RegistryRecordRow[]),
"registry",
);
console.log(`[registry:seed] Embedded ${chunkCount} registry corpus chunk(s).`);
}
}

Expand Down
14 changes: 6 additions & 8 deletions src/lib/differential-seed.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { buildDefaultDifferentialRows } from "@/lib/differential-fixtures";
import { type DifferentialRecordInsert, type DifferentialRecordRow } from "@/lib/differential-records";
import { embedDifferentialRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { bestEffortEmbedRows, embedDifferentialRows } from "@/lib/registry-corpus";

type AdminClient = ReturnType<typeof import("@/lib/supabase/admin").createAdminClient>;

Expand All @@ -17,13 +17,11 @@ export async function ensureDifferentialsSeeded(
.select("*");
if (error) throw new Error(`Differential seed failed: ${error.message}`);
const seededRows = (data ?? []) as DifferentialRecordRow[];
if (registryCorpusEmbeddingEnabled()) {
try {
await embedDifferentialRows(supabase, seededRows);
} catch (embedError) {
console.error(`[differentials] corpus embedding failed for owner ${ownerId}`, embedError);
}
}
await bestEffortEmbedRows({
scope: "differentials",
ownerId,
embed: () => embedDifferentialRows(supabase, seededRows),
});
return seededRows;
}

Expand Down
14 changes: 6 additions & 8 deletions src/lib/medication-seed.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { buildDefaultMedicationRows, defaultMedicationRecords } from "@/lib/medication-fixtures";
import { embedMedicationRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { bestEffortEmbedRows, embedMedicationRows } from "@/lib/registry-corpus";
import { type MedicationRecordInsert, type MedicationRecordRow } from "@/lib/medication-records";

type AdminClient = ReturnType<typeof import("@/lib/supabase/admin").createAdminClient>;
Expand All @@ -16,13 +16,11 @@ export async function ensureMedicationsSeeded(supabase: AdminClient, ownerId: st
.select("*");
if (error) throw new Error(`Medication seed failed: ${error.message}`);
const seededRows = (data ?? []) as MedicationRecordRow[];
if (registryCorpusEmbeddingEnabled()) {
try {
await embedMedicationRows(supabase, seededRows);
} catch (embedError) {
console.error(`[medications] corpus embedding failed for owner ${ownerId}`, embedError);
}
}
await bestEffortEmbedRows({
scope: "medications",
ownerId,
embed: () => embedMedicationRows(supabase, seededRows),
});
return seededRows;
}

Expand Down
27 changes: 10 additions & 17 deletions src/lib/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2376,9 +2376,8 @@ async function attachDocumentRankingMetadata(
for (const row of metadataRows) {
cache.documentMetadata.set(row.document_id, { labels: row.labels, summary: row.summary });
}
const metadataByDocument = new Map(metadataRows.map((row) => [row.document_id, row]));
const enriched = results.map((result) => {
const metadata = metadataByDocument.get(result.document_id) ?? cache.documentMetadata.get(result.document_id);
const metadata = cache.documentMetadata.get(result.document_id);
if (!metadata) return result;
return {
...result,
Expand All @@ -2392,6 +2391,13 @@ async function attachDocumentRankingMetadata(
}
}

function withCachedIndexQuality(results: SearchResult[], cache: DocumentRankingMetadataCache) {
return results.map((result) => ({
...result,
indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null,
}));
}

async function attachIndexQualityMetadata(
supabase: ReturnType<typeof createAdminClient>,
results: SearchResult[],
Expand All @@ -2401,12 +2407,7 @@ async function attachIndexQualityMetadata(
const documentIds = Array.from(new Set(results.map((result) => result.document_id)));
if (documentIds.length === 0) return results;
const missingDocumentIds = documentIds.filter((documentId) => !cache.indexQuality.has(documentId));
if (missingDocumentIds.length === 0) {
return results.map((result) => ({
...result,
indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null,
}));
}
if (missingDocumentIds.length === 0) return withCachedIndexQuality(results, cache);
try {
let query = supabase
.from("document_index_quality")
Expand All @@ -2417,15 +2418,7 @@ async function attachIndexQualityMetadata(
if (error) return results;
for (const documentId of missingDocumentIds) cache.indexQuality.set(documentId, null);
for (const row of data ?? []) cache.indexQuality.set(row.document_id, row as SearchResult["indexing_quality"]);
const qualityByDocument = new Map((data ?? []).map((row) => [row.document_id, row]));
return results.map((result) => ({
...result,
indexing_quality:
(qualityByDocument.get(result.document_id) as SearchResult["indexing_quality"]) ??
cache.indexQuality.get(result.document_id) ??
result.indexing_quality ??
null,
}));
return withCachedIndexQuality(results, cache);
} catch {
return results;
}
Expand Down
47 changes: 39 additions & 8 deletions src/lib/registry-corpus.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,17 @@ function registryBaseMetadata(entry: RegistryCorpusEntry): Record<string, Json>
};
}

function registryDocumentId(entry: RegistryCorpusEntry) {
return deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`);
}

function registryEntryMetadata(entry: RegistryCorpusEntry): Record<string, Json> {
return { ...registryBaseMetadata(entry), ...entry.metadata };
}

function registryDocumentRow(entry: RegistryCorpusEntry): TablesInsert<"documents"> {
const documentId = deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`);
const metadata = { ...registryBaseMetadata(entry), ...entry.metadata };
return {
id: documentId,
id: registryDocumentId(entry),
owner_id: entry.ownerId,
title: entry.title,
description: entry.subtitle,
Expand All @@ -98,16 +104,14 @@ function registryDocumentRow(entry: RegistryCorpusEntry): TablesInsert<"document
chunk_count: 1,
image_count: 0,
content_hash: sha256(entry.content),
metadata,
metadata: registryEntryMetadata(entry),
};
}

function registryChunkRow(entry: RegistryCorpusEntry, embedding: Vector): TablesInsert<"document_chunks"> {
const documentId = deterministicUuid(`registry-document:${entry.kind}:${entry.recordId}`);
const metadata = { ...registryBaseMetadata(entry), ...entry.metadata };
return {
id: deterministicUuid(`registry-chunk:${entry.kind}:${entry.recordId}`),
document_id: documentId,
document_id: registryDocumentId(entry),
chunk_index: 0,
page_number: 1,
section_heading: "Registry summary",
Expand All @@ -118,7 +122,7 @@ function registryChunkRow(entry: RegistryCorpusEntry, embedding: Vector): Tables
image_ids: [],
content_hash: sha256(entry.content),
embedding,
metadata,
metadata: registryEntryMetadata(entry),
};
}

Expand Down Expand Up @@ -275,3 +279,30 @@ export function embedMedicationRows(supabase: AdminClient, rows: MedicationRecor
export function embedDifferentialRows(supabase: AdminClient, rows: DifferentialRecordRow[]) {
return embedRegistryCorpusEntries(supabase, differentialRowsToCorpusEntries(rows));
}

/** Best-effort corpus embedding for a seed path: failures are logged, not thrown,
* so a broken embedding call never blocks the seed write it runs after. */
export async function bestEffortEmbedRows(args: { scope: string; ownerId: string; detail?: string; embed: () => Promise<unknown> }) {
if (!registryCorpusEmbeddingEnabled()) return;
try {
await args.embed();
} catch (embedError) {
const suffix = args.detail ? ` ${args.detail}` : "";
console.error(`[${args.scope}] corpus embedding failed for owner ${args.ownerId}${suffix}`, embedError);
}
}

/** Reload an owner's rows for a table and embed them, returning the chunk count
* written. Shared by the registry/medication/differential seed CLIs, which each
* re-read their own rows (rather than reusing the upsert response) so the
* embedded corpus always reflects what is actually stored. */
export async function embedReloadedOwnerRows<Row>(
reload: PromiseLike<{ data: Row[] | null; error: { message: string } | null }>,
embed: (rows: Row[]) => Promise<{ chunkCount: number }>,
tableLabel: string,
) {
const { data, error } = await reload;
if (error) throw new Error(`Could not reload ${tableLabel} rows for embedding: ${error.message}`);
const { chunkCount } = await embed(data ?? []);
return chunkCount;
}
15 changes: 7 additions & 8 deletions src/lib/registry-seed.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { formRecords } from "@/lib/forms";
import { buildDefaultFormRows, buildDefaultServiceRows, defaultServiceRecords } from "@/lib/registry-fixtures";
import { embedClinicalRegistryRows, registryCorpusEmbeddingEnabled } from "@/lib/registry-corpus";
import { bestEffortEmbedRows, embedClinicalRegistryRows } from "@/lib/registry-corpus";
import { type RegistryRecordInsert, type RegistryRecordKind, type RegistryRecordRow } from "@/lib/registry-records";

// Type-only reference to the admin client so this module carries no runtime
Expand Down Expand Up @@ -44,13 +44,12 @@ export async function ensureRegistrySeeded(
.select("*");
if (error) throw new Error(`Registry seed failed: ${error.message}`);
const seededRows = (data ?? []) as RegistryRecordRow[];
if (registryCorpusEmbeddingEnabled()) {
try {
await embedClinicalRegistryRows(supabase, seededRows);
} catch (embedError) {
console.error(`[registry] corpus embedding failed for owner ${ownerId} (${kind})`, embedError);
}
}
await bestEffortEmbedRows({
scope: "registry",
ownerId,
detail: `(${kind})`,
embed: () => embedClinicalRegistryRows(supabase, seededRows),
});
return seededRows;
}

Expand Down