diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index d5b7bb6e3..fba23c9b8 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -292,6 +292,7 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin cheapImageSkipReason, classifiedImageSkipReason, clinicalImagePolicyVersion, + imagePlacementDedupeKey, lightweightPerceptualHash, normalizeImageBbox, }, @@ -332,17 +333,19 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin const bytes = Buffer.from(await download.data.arrayBuffer()); const imageHash = image.image_hash ?? hashBytes(bytes); const perceptualHash = lightweightPerceptualHash(imageHash, image.width, image.height); + const filterImage = { + bbox: normalizeImageBbox(image.bbox), + width: image.width, + height: image.height, + pageNumber: typeof image.page_number === "number" ? image.page_number : undefined, + sourceKind: image.source_kind as + "embedded" | "table_crop" | "diagram_crop" | "page_region" | "fallback" | undefined, + }; const cheapSkip = cheapImageSkipReason({ bytesLength: bytes.length, imageHash, seenHashes, - image: { - bbox: normalizeImageBbox(image.bbox), - width: image.width, - height: image.height, - sourceKind: image.source_kind as - "embedded" | "table_crop" | "diagram_crop" | "page_region" | "fallback" | undefined, - }, + image: filterImage, }); if (cheapSkip) { @@ -359,7 +362,8 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin skipped += 1; continue; } - seenHashes.add(imageHash); + const placementKey = imagePlacementDedupeKey({ imageHash, image: filterImage }); + if (placementKey) seenHashes.add(placementKey); const baseAssessment = assessClinicalImageUse({ imageType: image.image_type, diff --git a/src/lib/image-filtering.ts b/src/lib/image-filtering.ts index 5ae1d3f69..967ee60d6 100644 --- a/src/lib/image-filtering.ts +++ b/src/lib/image-filtering.ts @@ -9,7 +9,7 @@ export type CheapImageFilterInput = { bytesLength: number; imageHash: string; seenHashes: Set; - image: Pick; + image: Pick & Partial>; }; export type ClassifiedImage = { @@ -320,6 +320,31 @@ export function normalizeImageBbox(value: unknown): [number, number, number, num return coords.every((coord): coord is number => coord !== null) ? (coords as [number, number, number, number]) : null; } +function stableBboxKey(bbox: unknown): string | null { + const coords = normalizeImageBbox(bbox); + // Missing/malformed bboxes must not share one sentinel (e.g. "bbox:null"): that would + // falsely collapse distinct unknown placements on the same page into a single de-dupe key. + // Only comparable coordinates participate in cheap placement de-dupe. + return coords ? coords.map((coord) => Number(coord.toFixed(2))).join(",") : null; +} + +// Exact byte hash alone is too broad for ingestion de-dupe: some PDF extractors can emit +// visually identical table/diagram crops from different page contexts, and dropping every +// later occurrence loses operator-visible clinical evidence. Limit cheap de-dupe to the +// same extracted placement; header/footer/logo filters still remove repeated decorative +// assets across pages. Returns null when bbox placement is not comparable so callers can +// skip de-dupe rather than collide unknown placements. +export function imagePlacementDedupeKey(input: { + imageHash: string; + image: Pick & Partial>; +}): string | null { + const bboxKey = stableBboxKey(input.image.bbox); + if (bboxKey === null) return null; + return [input.imageHash, input.image.sourceKind ?? "embedded", input.image.pageNumber ?? "page:null", bboxKey].join( + "|", + ); +} + function bboxLooksLikeHeaderOrFooter(bbox: unknown) { const coords = normalizeImageBbox(bbox); if (!coords) return false; @@ -335,7 +360,9 @@ export function cheapImageSkipReason(input: CheapImageFilterInput) { const width = image.width ?? null; const height = image.height ?? null; - if (seenHashes.has(imageHash)) return "duplicate image"; + const placementKey = imagePlacementDedupeKey({ imageHash, image }); + // Only de-dupe when placement is comparable; unknown/malformed bboxes stay unique. + if (placementKey !== null && seenHashes.has(placementKey)) return "duplicate image"; if (sourceKind === "embedded" && bytesLength < 4096) return "small decorative image"; if (width && height) { const shortestSide = Math.min(width, height); diff --git a/tests/image-filtering.test.ts b/tests/image-filtering.test.ts index a948b33e2..49af17a39 100644 --- a/tests/image-filtering.test.ts +++ b/tests/image-filtering.test.ts @@ -4,6 +4,7 @@ import { cheapImageSkipReason, classifiedImageSkipReason, isClinicalImageEvidence, + imagePlacementDedupeKey, lightweightPerceptualHash, normalizeImageBbox, partitionViewerImages, @@ -11,13 +12,107 @@ import { describe("smart image filtering", () => { it("skips repeated exact image hashes before captioning", () => { - const seenHashes = new Set(["abc"]); + const image = { + sourceKind: "embedded" as const, + width: 600, + height: 400, + pageNumber: 1, + bbox: [10, 20, 610, 420] as [number, number, number, number], + }; + const placementKey = imagePlacementDedupeKey({ imageHash: "abc", image }); + expect(placementKey).toBeTruthy(); + const seenHashes = new Set([placementKey!]); expect( cheapImageSkipReason({ bytesLength: 80_000, imageHash: "abc", seenHashes, - image: { sourceKind: "embedded", width: 600, height: 400 }, + image, + }), + ).toBe("duplicate image"); + }); + + it("does not drop an exact-byte clinical image from a different page placement", () => { + const firstImage = { + sourceKind: "table_crop" as const, + width: 600, + height: 400, + pageNumber: 1, + bbox: [10, 20, 610, 420] as [number, number, number, number], + }; + const secondImage = { ...firstImage, pageNumber: 2 }; + const placementKey = imagePlacementDedupeKey({ imageHash: "same-bytes", image: firstImage }); + expect(placementKey).toBeTruthy(); + const seenHashes = new Set([placementKey!]); + + expect( + cheapImageSkipReason({ + bytesLength: 80_000, + imageHash: "same-bytes", + seenHashes, + image: secondImage, + }), + ).toBeNull(); + }); + + it("does not collapse missing or malformed bboxes into one shared de-dupe key", () => { + const base = { + sourceKind: "embedded" as const, + width: 600, + height: 400, + pageNumber: 1, + }; + const missingBbox = { ...base, bbox: null as unknown as [number, number, number, number] }; + const malformedBbox = { + ...base, + bbox: { x0: 10, y0: 20, x1: 610, y1: 420 } as unknown as [number, number, number, number], + }; + + expect(imagePlacementDedupeKey({ imageHash: "same-bytes", image: missingBbox })).toBeNull(); + expect(imagePlacementDedupeKey({ imageHash: "same-bytes", image: malformedBbox })).toBeNull(); + + const seenHashes = new Set(); + expect( + cheapImageSkipReason({ + bytesLength: 80_000, + imageHash: "same-bytes", + seenHashes, + image: missingBbox, + }), + ).toBeNull(); + // Even if a caller mistakenly records a sentinel, unknown placements must not de-dupe. + seenHashes.add("same-bytes|embedded|1|bbox:null"); + expect( + cheapImageSkipReason({ + bytesLength: 80_000, + imageHash: "same-bytes", + seenHashes, + image: malformedBbox, + }), + ).toBeNull(); + }); + + it("still de-dupes true identical placements when bbox coordinates match", () => { + const image = { + sourceKind: "table_crop" as const, + width: 600, + height: 400, + pageNumber: 3, + bbox: [12.345, 20.001, 610.004, 420.009] as [number, number, number, number], + }; + const samePlacement = { + ...image, + // Rounding-equivalent coords must share the placement key. + bbox: [12.35, 20, 610, 420.01] as [number, number, number, number], + }; + const placementKey = imagePlacementDedupeKey({ imageHash: "table-bytes", image }); + expect(placementKey).toBe(imagePlacementDedupeKey({ imageHash: "table-bytes", image: samePlacement })); + expect( + cheapImageSkipReason({ + bytesLength: 80_000, + imageHash: "table-bytes", + seenHashes: new Set([placementKey!]), + image: samePlacement, }), ).toBe("duplicate image"); }); diff --git a/worker/main.ts b/worker/main.ts index bef0660d9..0f035de3f 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -24,6 +24,7 @@ import { clinicalImagePolicyVersion, lowSignalImageTextSkipReason, lightweightPerceptualHash, + imagePlacementDedupeKey, } from "../src/lib/image-filtering"; import { isPartialIndexWriteConflict, @@ -857,7 +858,7 @@ async function uploadAndCaptionImages( cropCompleteness: number; ocrTextDensity: number; }> = []; - const seenHashes = new Set(); + const seenImagePlacements = new Set(); let skippedImages = 0; const skipReasons = new Map(); const imageTypeCounts = new Map(); @@ -917,7 +918,7 @@ async function uploadAndCaptionImages( const skipReason = cheapImageSkipReason({ bytesLength: preparedImage.bytesLength, imageHash, - seenHashes, + seenHashes: seenImagePlacements, image, }); if (skipReason) { @@ -925,7 +926,8 @@ async function uploadAndCaptionImages( noteSkippedImage(skipReasons, skipReason); continue; } - seenHashes.add(imageHash); + const placementKey = imagePlacementDedupeKey({ imageHash, image }); + if (placementKey) seenImagePlacements.add(placementKey); const nearbyText = image.pageNumber ? pagesByNumber.get(image.pageNumber) : undefined; const tableMetadata = imageTableMetadata(image);