fix(worker): preserve distinct image placements in cheap image de-duplication

scripts/enrich-documents.ts

@@ -292,6 +292,7 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
       cheapImageSkipReason,
       classifiedImageSkipReason,
       clinicalImagePolicyVersion,
+      imagePlacementDedupeKey,
       lightweightPerceptualHash,
       normalizeImageBbox,
     },
@@ -332,17 +333,19 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
     const bytes = Buffer.from(await download.data.arrayBuffer());
     const imageHash = image.image_hash ?? hashBytes(bytes);
     const perceptualHash = lightweightPerceptualHash(imageHash, image.width, image.height);
+    const filterImage = {
+      bbox: normalizeImageBbox(image.bbox),
+      width: image.width,
+      height: image.height,
+      pageNumber: typeof image.page_number === "number" ? image.page_number : undefined,
+      sourceKind: image.source_kind as
+        "embedded" | "table_crop" | "diagram_crop" | "page_region" | "fallback" | undefined,
+    };
     const cheapSkip = cheapImageSkipReason({
       bytesLength: bytes.length,
       imageHash,
       seenHashes,
-      image: {
-        bbox: normalizeImageBbox(image.bbox),
-        width: image.width,
-        height: image.height,
-        sourceKind: image.source_kind as
-          "embedded" | "table_crop" | "diagram_crop" | "page_region" | "fallback" | undefined,
-      },
+      image: filterImage,
     });
 
     if (cheapSkip) {
@@ -359,7 +362,8 @@ async function classifyExistingImages(supabase: SupabaseAdmin, documentId: strin
       skipped += 1;
       continue;
     }
-    seenHashes.add(imageHash);
+    const placementKey = imagePlacementDedupeKey({ imageHash, image: filterImage });
+    if (placementKey) seenHashes.add(placementKey);
 
     const baseAssessment = assessClinicalImageUse({
       imageType: image.image_type,

src/lib/image-filtering.ts

@@ -9,7 +9,7 @@ export type CheapImageFilterInput = {
   bytesLength: number;
   imageHash: string;
   seenHashes: Set<string>;
-  image: Pick<ExtractedImage, "bbox" | "height" | "width" | "sourceKind">;
+  image: Pick<ExtractedImage, "bbox" | "height" | "width" | "sourceKind"> & Partial<Pick<ExtractedImage, "pageNumber">>;
 };
 
 export type ClassifiedImage = {
@@ -320,6 +320,31 @@ export function normalizeImageBbox(value: unknown): [number, number, number, num
   return coords.every((coord): coord is number => coord !== null) ? (coords as [number, number, number, number]) : null;
 }
 
+function stableBboxKey(bbox: unknown): string | null {
+  const coords = normalizeImageBbox(bbox);
+  // Missing/malformed bboxes must not share one sentinel (e.g. "bbox:null"): that would
+  // falsely collapse distinct unknown placements on the same page into a single de-dupe key.
+  // Only comparable coordinates participate in cheap placement de-dupe.
+  return coords ? coords.map((coord) => Number(coord.toFixed(2))).join(",") : null;
+}
+
+// Exact byte hash alone is too broad for ingestion de-dupe: some PDF extractors can emit
+// visually identical table/diagram crops from different page contexts, and dropping every
+// later occurrence loses operator-visible clinical evidence. Limit cheap de-dupe to the
+// same extracted placement; header/footer/logo filters still remove repeated decorative
+// assets across pages. Returns null when bbox placement is not comparable so callers can
+// skip de-dupe rather than collide unknown placements.
+export function imagePlacementDedupeKey(input: {
+  imageHash: string;
+  image: Pick<ExtractedImage, "bbox" | "sourceKind"> & Partial<Pick<ExtractedImage, "pageNumber">>;
+}): string | null {
+  const bboxKey = stableBboxKey(input.image.bbox);
+  if (bboxKey === null) return null;
+  return [input.imageHash, input.image.sourceKind ?? "embedded", input.image.pageNumber ?? "page:null", bboxKey].join(
+    "|",
+  );
+}
+
 function bboxLooksLikeHeaderOrFooter(bbox: unknown) {
   const coords = normalizeImageBbox(bbox);
   if (!coords) return false;
@@ -335,7 +360,9 @@ export function cheapImageSkipReason(input: CheapImageFilterInput) {
   const width = image.width ?? null;
   const height = image.height ?? null;
 
-  if (seenHashes.has(imageHash)) return "duplicate image";
+  const placementKey = imagePlacementDedupeKey({ imageHash, image });
+  // Only de-dupe when placement is comparable; unknown/malformed bboxes stay unique.
+  if (placementKey !== null && seenHashes.has(placementKey)) return "duplicate image";
   if (sourceKind === "embedded" && bytesLength < 4096) return "small decorative image";
   if (width && height) {
     const shortestSide = Math.min(width, height);

tests/image-filtering.test.ts

@@ -4,20 +4,115 @@ import {
   cheapImageSkipReason,
   classifiedImageSkipReason,
   isClinicalImageEvidence,
+  imagePlacementDedupeKey,
   lightweightPerceptualHash,
   normalizeImageBbox,
   partitionViewerImages,
 } from "../src/lib/image-filtering";
 
 describe("smart image filtering", () => {
   it("skips repeated exact image hashes before captioning", () => {
-    const seenHashes = new Set(["abc"]);
+    const image = {
+      sourceKind: "embedded" as const,
+      width: 600,
+      height: 400,
+      pageNumber: 1,
+      bbox: [10, 20, 610, 420] as [number, number, number, number],
+    };
+    const placementKey = imagePlacementDedupeKey({ imageHash: "abc", image });
+    expect(placementKey).toBeTruthy();
+    const seenHashes = new Set([placementKey!]);
     expect(
       cheapImageSkipReason({
         bytesLength: 80_000,
         imageHash: "abc",
         seenHashes,
-        image: { sourceKind: "embedded", width: 600, height: 400 },
+        image,
+      }),
+    ).toBe("duplicate image");
+  });
+
+  it("does not drop an exact-byte clinical image from a different page placement", () => {
+    const firstImage = {
+      sourceKind: "table_crop" as const,
+      width: 600,
+      height: 400,
+      pageNumber: 1,
+      bbox: [10, 20, 610, 420] as [number, number, number, number],
+    };
+    const secondImage = { ...firstImage, pageNumber: 2 };
+    const placementKey = imagePlacementDedupeKey({ imageHash: "same-bytes", image: firstImage });
+    expect(placementKey).toBeTruthy();
+    const seenHashes = new Set([placementKey!]);
+
+    expect(
+      cheapImageSkipReason({
+        bytesLength: 80_000,
+        imageHash: "same-bytes",
+        seenHashes,
+        image: secondImage,
+      }),
+    ).toBeNull();
+  });
+
+  it("does not collapse missing or malformed bboxes into one shared de-dupe key", () => {
+    const base = {
+      sourceKind: "embedded" as const,
+      width: 600,
+      height: 400,
+      pageNumber: 1,
+    };
+    const missingBbox = { ...base, bbox: null as unknown as [number, number, number, number] };
+    const malformedBbox = {
+      ...base,
+      bbox: { x0: 10, y0: 20, x1: 610, y1: 420 } as unknown as [number, number, number, number],
+    };
+
+    expect(imagePlacementDedupeKey({ imageHash: "same-bytes", image: missingBbox })).toBeNull();
+    expect(imagePlacementDedupeKey({ imageHash: "same-bytes", image: malformedBbox })).toBeNull();
+
+    const seenHashes = new Set<string>();
+    expect(
+      cheapImageSkipReason({
+        bytesLength: 80_000,
+        imageHash: "same-bytes",
+        seenHashes,
+        image: missingBbox,
+      }),
+    ).toBeNull();
+    // Even if a caller mistakenly records a sentinel, unknown placements must not de-dupe.
+    seenHashes.add("same-bytes|embedded|1|bbox:null");
+    expect(
+      cheapImageSkipReason({
+        bytesLength: 80_000,
+        imageHash: "same-bytes",
+        seenHashes,
+        image: malformedBbox,
+      }),
+    ).toBeNull();
+  });
+
+  it("still de-dupes true identical placements when bbox coordinates match", () => {
+    const image = {
+      sourceKind: "table_crop" as const,
+      width: 600,
+      height: 400,
+      pageNumber: 3,
+      bbox: [12.345, 20.001, 610.004, 420.009] as [number, number, number, number],
+    };
+    const samePlacement = {
+      ...image,
+      // Rounding-equivalent coords must share the placement key.
+      bbox: [12.35, 20, 610, 420.01] as [number, number, number, number],
+    };
+    const placementKey = imagePlacementDedupeKey({ imageHash: "table-bytes", image });
+    expect(placementKey).toBe(imagePlacementDedupeKey({ imageHash: "table-bytes", image: samePlacement }));
+    expect(
+      cheapImageSkipReason({
+        bytesLength: 80_000,
+        imageHash: "table-bytes",
+        seenHashes: new Set([placementKey!]),
+        image: samePlacement,
       }),
     ).toBe("duplicate image");
   });

worker/main.ts

@@ -24,6 +24,7 @@ import {
   clinicalImagePolicyVersion,
   lowSignalImageTextSkipReason,
   lightweightPerceptualHash,
+  imagePlacementDedupeKey,
 } from "../src/lib/image-filtering";
 import {
   isPartialIndexWriteConflict,
@@ -857,7 +858,7 @@ async function uploadAndCaptionImages(
     cropCompleteness: number;
     ocrTextDensity: number;
   }> = [];
-  const seenHashes = new Set<string>();
+  const seenImagePlacements = new Set<string>();
   let skippedImages = 0;
   const skipReasons = new Map<string, number>();
   const imageTypeCounts = new Map<string, number>();
@@ -917,15 +918,16 @@ async function uploadAndCaptionImages(
     const skipReason = cheapImageSkipReason({
       bytesLength: preparedImage.bytesLength,
       imageHash,
-      seenHashes,
+      seenHashes: seenImagePlacements,
       image,
     });
     if (skipReason) {
       skippedImages += 1;
       noteSkippedImage(skipReasons, skipReason);
       continue;
     }
-    seenHashes.add(imageHash);
+    const placementKey = imagePlacementDedupeKey({ imageHash, image });
+    if (placementKey) seenImagePlacements.add(placementKey);
 
     const nearbyText = image.pageNumber ? pagesByNumber.get(image.pageNumber) : undefined;
     const tableMetadata = imageTableMetadata(image);