From 3e4544d852e18aa225902f865c076335c664270c Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Thu, 25 Jun 2026 17:46:10 +0000 Subject: [PATCH] fix(recipes): exclude Llama-4 vision branch from default PTQ quantization The default_disabled_quantizers unit kept the multimodal vision branch in BF16 via the patterns *embed_vision* / *vision_tower* / *visual*, which cover gemma / Qwen-VL / Kimi naming but miss Llama-4, whose encoder is named vision_model and whose projector is multi_modal_projector. As a result general/ptq/nvfp4_default-kv_fp8 (and any recipe importing this unit) quantized the Llama-4 vision tower. Export then failed on vision_model.patch_embedding.linear, whose in_features=588 (3*14*14) is not divisible by the NVFP4 block size: AssertionError: Weight shape is not divisible for block size for block quantization. Add *vision_model* and *multi_modal_projector* to the disabled-quantizer patterns so the Llama-4 vision branch stays in BF16 by default, matching the existing behavior for other VL models. Fixes NVBugs 6359097. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Zhiyu Cheng --- .../ptq/units/default_disabled_quantizers.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml index 87fd67300fa..5e48dc73b7e 100644 --- a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml +++ b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml @@ -43,10 +43,13 @@ # Multimodal vision branch: keep the vision encoder (SigLIP / ViT) and any # multimodal embedding projection in BF16 by default. Recipes that enable bare # `*weight_quantizer` / `*input_quantizer` or `*mlp*` wildcards otherwise also - # match the vision tower (`model.vision_tower.*`, `model.visual.*`) and the - # embedding projection (`model.embed_vision.*`); quantizing the vision branch - # crashes export / produces garbage image embeddings on VL models (gemma-4, - # Qwen3.5-VL — NVBugs 6293731, 6293762, 6294017). A recipe that intentionally + # match the vision tower (`model.vision_tower.*`, `model.visual.*`, + # Llama-4 `vision_model.*`) and the embedding projection (`model.embed_vision.*`, + # Llama-4 `multi_modal_projector.*`); quantizing the vision branch crashes + # export / produces garbage image embeddings on VL models (gemma-4, + # Qwen3.5-VL — NVBugs 6293731, 6293762, 6294017; Llama-4-Scout — NVBugs + # 6359097, where `vision_model.patch_embedding.linear` has in_features=588, + # not divisible by the NVFP4 block size). A recipe that intentionally # quantizes vision must re-enable these after importing this unit. - quantizer_name: '*embed_vision*' enable: false @@ -54,6 +57,10 @@ enable: false - quantizer_name: '*visual*' enable: false + - quantizer_name: '*vision_model*' + enable: false + - quantizer_name: '*multi_modal_projector*' + enable: false - parent_class: 'nn.BatchNorm1d' quantizer_name: '*' enable: false