From f88bc9fd992fce815d53f15fab9152dfc05649d7 Mon Sep 17 00:00:00 2001 From: Mallory M Date: Sun, 12 Jul 2026 13:39:01 -0700 Subject: [PATCH] fix: denormalize boxes against the original image dims, not the padded canvas Engine::locate_image projected coordinate tokens onto the 28px-padded preprocess canvas (P.target_w/target_h) instead of the original image dims. The reference implementation denormalizes <0>..<1000> against image.size, so on any non-28-aligned input every box was stretched by up to one pad interval and could overshoot the image bounds (e.g. 1024x576 input: token y2=<998> came out 586.8px on a 576px image; official emits 574.8). The parity suite missed it twice: the only fixture is 448x448 (28-aligned, so padded==original) and the dump scripts + diff_upstream.py projected the reference boxes onto the same padded grid dims, so both sides agreed with each other while both disagreed with the official pixel output. - src/engine.cpp: pass img.w/img.h to parse_boxes. - scripts/{dump_slow,dump_fast,dump_preproc}_reference.py, diff_upstream.py: project reference boxes onto the original image size (regenerate dumps/; with regenerated dumps the existing bus.jpg variable-grid gate in test_engine guards this convention). - tests/test_engine.cpp: assert all boxes lie within the original image bounds. Validated against the official PyTorch implementation on 60 (frame, phrase) pairs of real 1024x576 photos: wherever the generated token streams agree, pixel boxes now match the official output to <0.01px; before this change they differed by up to ~12px. Co-Authored-By: Claude Fable 5 --- benchmarks/bench.py | 4 ++-- benchmarks/demo/dgx_measure.py | 4 ++-- benchmarks/demo/dgx_official.py | 3 ++- scripts/diff_upstream.py | 12 ++++++------ scripts/dump_fast_reference.py | 6 ++++-- scripts/dump_preproc_reference.py | 7 +++++-- scripts/dump_slow_reference.py | 6 ++++-- src/engine.cpp | 8 +++++--- tests/test_engine.cpp | 15 +++++++++++++++ 9 files changed, 45 insertions(+), 20 deletions(-) diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 6162206..1eb6da4 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -76,8 +76,8 @@ def upstream_time(bundle, image, prompt, mode, max_new=256): tokenizer=tok, max_new_tokens=max_new, generation_mode=mode, use_cache=True) dt = time.time() - t0 ids = tok(s, add_special_tokens=False)["input_ids"] - grid = inputs["image_grid_hws"][0].tolist() - boxes = parse_boxes_from_ids(ids, grid[1]*14, grid[0]*14) + from PIL import Image + boxes = parse_boxes_from_ids(ids, *Image.open(image).size) # ORIGINAL dims (reference convention) return dt, boxes diff --git a/benchmarks/demo/dgx_measure.py b/benchmarks/demo/dgx_measure.py index e9878be..425a318 100644 --- a/benchmarks/demo/dgx_measure.py +++ b/benchmarks/demo/dgx_measure.py @@ -48,8 +48,8 @@ def gen(): if device == "cuda": torch.cuda.synchronize() dt = time.time() - t0 ids = tok(s, add_special_tokens=False)["input_ids"] - g = inputs["image_grid_hws"][0].tolist() - boxes = _dedup(parse_boxes_from_ids(ids, g[1]*14, g[0]*14)) + from PIL import Image + boxes = _dedup(parse_boxes_from_ids(ids, *Image.open(IMG).size)) # ORIGINAL dims del model; torch.cuda.empty_cache() if device == "cuda" else None return round(dt, 2), len(ids), boxes diff --git a/benchmarks/demo/dgx_official.py b/benchmarks/demo/dgx_official.py index 4fbaeda..d53ff8f 100644 --- a/benchmarks/demo/dgx_official.py +++ b/benchmarks/demo/dgx_official.py @@ -109,7 +109,8 @@ def measure_upstream(model, tok, proc, images): res = {} for img, cats in images: inputs = up_inputs(proc, img, cats) - g = inputs["image_grid_hws"][0].tolist(); tw, th = g[1]*14, g[0]*14 + from PIL import Image as _PILImage + tw, th = _PILImage.open(img).size # ORIGINAL dims (reference convention) # --- greedy (deterministic; parity + fair-work timing, max_new=256) --- s = gen_call(model, tok, inputs, max_new_tokens=256, do_sample=False); torch.cuda.synchronize() tg = [] diff --git a/scripts/diff_upstream.py b/scripts/diff_upstream.py index 75d0179..96d74be 100644 --- a/scripts/diff_upstream.py +++ b/scripts/diff_upstream.py @@ -17,7 +17,8 @@ * Boxes are parsed from upstream's generated token stream with the SAME token-id -> coord -> pixel logic as ``dump_slow_reference.py`` (which our C++ ``boxes.cpp`` is gated to reproduce). Both sides denormalize against the - preprocessed grid size (gw*14 x gh*14), so the comparison is apples-to-apples. + ORIGINAL image size — the reference convention (the model card demo projects + <0>..<1000> onto image.size) — so the comparison is apples-to-apples. * Both sides use ``max_new_tokens=256`` (the C++ engine/CLI default) so the hybrid degenerate tail is the same length on both. @@ -99,7 +100,7 @@ def build_inputs_for(model_bundle, image_path, prompt, device="cpu"): def parse_boxes_from_ids(ids, img_w, img_h): - """Token-id -> [(label, [x1,y1,x2,y2]) ...] in grid-pixel space. + """Token-id -> [(label, [x1,y1,x2,y2]) ...] in original-image pixel space. Identical logic to dump_slow_reference.py / src/boxes.cpp (x1,y1,x2,y2).""" boxes, i, n, cur_label = [], 0, len(ids), "" while i < n: @@ -141,8 +142,7 @@ def upstream_boxes(model_bundle, image_path, prompt, mode, max_new=256): attention_mask=inputs["attention_mask"], image_grid_hws=inputs["image_grid_hws"], tokenizer=tok, max_new_tokens=max_new, generation_mode=mode, use_cache=True) ids = tok(s, add_special_tokens=False)["input_ids"] - grid = inputs["image_grid_hws"][0].tolist() - img_w, img_h = grid[1] * 14, grid[0] * 14 # grid = [h_patches, w_patches] + img_w, img_h = Image.open(image_path).size # ORIGINAL dims (reference convention) return parse_boxes_from_ids(ids, img_w, img_h), (img_w, img_h) @@ -234,7 +234,7 @@ def main(): img, prompt, mode = c["image"], c["prompt"], c["mode"] name = Path(img).name try: - up, (gw, gh) = upstream_boxes(bundle, img, prompt, mode, args.max_new) + up, (iw, ih) = upstream_boxes(bundle, img, prompt, mode, args.max_new) cp = cpp_boxes(args.cli, args.gguf, img, prompt, mode) except Exception as e: print(f"[{idx}] {name:24s} {mode:6s} ERROR: {e}") @@ -248,7 +248,7 @@ def main(): tag = "OK " if ok else "DIFF" print(f"[{idx}] {name:24s} {mode:6s} {tag} up={len(up)} cpp={len(cp)} " f"matched={len(matched)} unmatched(up/cpp)={un_up}/{un_cp} " - f"minIoU={min_iou:.3f} maxCoordDiff={max_cd:.1f}px grid={gw}x{gh}") + f"minIoU={min_iou:.3f} maxCoordDiff={max_cd:.1f}px img={iw}x{ih}") if not ok: up_s = [(l, [round(x, 1) for x in b]) for l, b in up] cp_s = [(l, [round(x, 1) for x in b]) for l, b in cp] diff --git a/scripts/dump_fast_reference.py b/scripts/dump_fast_reference.py index d3c6160..c6cf1f4 100644 --- a/scripts/dump_fast_reference.py +++ b/scripts/dump_fast_reference.py @@ -30,8 +30,10 @@ def main(): ids = tok(s, add_special_tokens=False)["input_ids"] assert tok.decode(ids, skip_special_tokens=False) == s, "id round-trip mismatch" ids = np.array(ids, dtype=np.int64) - grid = inputs["image_grid_hws"][0].tolist() # [h_patches, w_patches] - img_h, img_w = grid[0]*14, grid[1]*14 # patches * patch_size + from PIL import Image + # ORIGINAL image dims — the reference denormalizes <0>..<1000> against + # image.size, not the 28px-padded preprocess canvas (grid*14). + img_w, img_h = Image.open(ROOT / spec["image"]).size boxes, labels = [], [] i, n, cur_label = 0, len(ids), "" while i < n: diff --git a/scripts/dump_preproc_reference.py b/scripts/dump_preproc_reference.py index 2eed747..5d20cca 100644 --- a/scripts/dump_preproc_reference.py +++ b/scripts/dump_preproc_reference.py @@ -65,6 +65,9 @@ def main(): img = Image.open(IMAGE_PATH).convert("RGB") print(f"image {IMAGE_PATH} size(w,h)={img.size}") + # Boxes denormalize against the ORIGINAL image dims (reference convention), + # not the padded target canvas. + orig_w, orig_h = img.size # =================== Manual _preprocess (capture resized_chw) =========== # Mirror LocateAnythingImageProcessor._preprocess exactly: @@ -170,8 +173,8 @@ def main(): if len(coords) == 4: x1, y1, x2, y2 = coords # x1,y1,x2,y2 boxes.append([ - x1 / 1000 * target_w, y1 / 1000 * target_h, - x2 / 1000 * target_w, y2 / 1000 * target_h, + x1 / 1000 * orig_w, y1 / 1000 * orig_h, + x2 / 1000 * orig_w, y2 / 1000 * orig_h, ]) labels.append(cur_label) i = j + 1 diff --git a/scripts/dump_slow_reference.py b/scripts/dump_slow_reference.py index 78e9d10..063698e 100644 --- a/scripts/dump_slow_reference.py +++ b/scripts/dump_slow_reference.py @@ -21,8 +21,10 @@ def main(): ids = tok(s, add_special_tokens=False)["input_ids"] assert tok.decode(ids, skip_special_tokens=False) == s, "id round-trip mismatch" ids = np.array(ids, dtype=np.int64) - grid = inputs["image_grid_hws"][0].tolist() # [h_patches, w_patches] - img_h, img_w = grid[0]*14, grid[1]*14 # patches * patch_size + from PIL import Image + # ORIGINAL image dims — the reference denormalizes <0>..<1000> against + # image.size, not the 28px-padded preprocess canvas (grid*14). + img_w, img_h = Image.open(ROOT / spec["image"]).size boxes, labels = [], [] i, n, cur_label = 0, len(ids), "" while i < n: diff --git a/src/engine.cpp b/src/engine.cpp index c743266..837e321 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -74,9 +74,11 @@ std::vector Engine::locate_image(const Image& img, const std::string& query /*early_stop=*/!std::getenv("LA_NO_EARLYSTOP")); if(!ok) return empty; - // 6) parse boxes. Coords denormalize against the preprocessed target size - // (img_w = target_w = gw*14, img_h = target_h = gh*14). - return parse_boxes(gen_ids, P.target_w, P.target_h, + // 6) parse boxes. Coords denormalize against the ORIGINAL image dims — the + // reference implementation projects <0>..<1000> tokens onto image.size, + // not the padded/downscaled preprocess canvas (which overshoots by up to + // one 28px pad and can exceed the image bounds). + return parse_boxes(gen_ids, img.w, img.h, [&](const std::vector& t){ return tok_.decode(t); }); } diff --git a/tests/test_engine.cpp b/tests/test_engine.cpp index e7a0538..8a1b5fe 100644 --- a/tests/test_engine.cpp +++ b/tests/test_engine.cpp @@ -1,9 +1,20 @@ #include "engine.hpp" +#include "image_io.hpp" #include "parity.hpp" #include #include #include #include + +// Boxes denormalize against the ORIGINAL image dims (reference convention). +// Regression guard: they were previously projected onto the 28px-padded +// preprocess canvas and could overshoot the image on non-aligned inputs. +static int boxes_within(const std::vector& bs, float w, float h){ + int ok = 1; + for(const auto& b : bs) + ok &= (b.x1>=-0.5f && b.y1>=-0.5f && b.x2<=w+0.5f && b.y2<=h+0.5f); + return ok; +} int main(){ const char* gguf=std::getenv("LA_TEST_GGUF"); const char* slow=std::getenv("LA_TEST_SLOW"); if(!gguf||!slow){std::fprintf(stderr,"skip\n");return 77;} @@ -23,6 +34,7 @@ int main(){ std::printf(" box[%d] %s [%.1f %.1f %.1f %.1f] ref [%.1f %.1f %.1f %.1f]\n", k, boxes[k].label.c_str(), boxes[k].x1,boxes[k].y1,boxes[k].x2,boxes[k].y2, r[0],r[1],r[2],r[3]); } + ok &= boxes_within(boxes, 448.f, 448.f); // Variable-grid (non-448) box gate on bus.jpg. The reference boxes + labels live // in reference_preproc.gguf (slow_boxes / la_preproc.box_labels). JPEG-decode noise @@ -60,6 +72,9 @@ int main(){ } else { std::printf("bus.jpg: no reference boxes in LA_TEST_PREPROC; smoke only (%zu boxes)\n", bb.size()); } + la::Image bimg; + if(la::load_image_rgb(bpath.empty()?std::string(bus):bpath, bimg)) + bok &= boxes_within(bb, (float)bimg.w, (float)bimg.h); } else { std::printf("bus.jpg: skip (LA_TEST_PREPROC unset or image missing)\n"); }