From ff759b3ecd5fa6619d63d916c0f0dc8c01fb5d5e Mon Sep 17 00:00:00 2001 From: realAsma Date: Tue, 30 Jun 2026 19:19:35 +0000 Subject: [PATCH 1/4] Refine DeciLM dtype handling in HF PTQ Signed-off-by: realAsma --- examples/hf_ptq/example_utils.py | 36 ++++++++++++--------- tests/examples/hf_ptq/test_example_utils.py | 2 +- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index dbb598038e5..1f166de8e97 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -617,6 +617,23 @@ def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs) return hf_config +def _get_config_dtype(config): + config_dtype = ( + getattr(config, "dtype", None) or getattr(config, "torch_dtype", None) or torch.bfloat16 + ) + if isinstance(config_dtype, str): + config_dtype = getattr(torch, config_dtype) + return config_dtype + + +def _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm): + model_kwargs = model_kwargs.copy() + if is_decilm: + model_kwargs["torch_dtype"] = config_dtype + model_kwargs.pop("dtype", None) + return model_kwargs + + def get_model( ckpt_path, device="cuda", @@ -758,20 +775,11 @@ def has_pack_quantized_config(config): with init_empty_weights(include_buffers=True): # When computing the device_map, assuming bfloat16 precision by default, # unless specified by the hf_config. - config_dtype = ( - getattr(config_for_init, "dtype", None) - or getattr(config_for_init, "torch_dtype", None) - or torch.bfloat16 - ) - if isinstance(config_dtype, str): - config_dtype = getattr(torch, config_dtype) - model_kwargs2 = model_kwargs.copy() + config_dtype = _get_config_dtype(config_for_init) + model_kwargs2 = _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm) if auto_model_module not in [AutoModelForCausalLM, AutoModel]: model_kwargs2.pop("trust_remote_code", None) - if is_decilm: - model_kwargs2["torch_dtype"] = config_dtype - model_kwargs2.pop("dtype", None) - else: + if not is_decilm: model_kwargs2["dtype"] = config_dtype model_kwargs2.pop("max_memory", None) model = from_config(config_for_init, **model_kwargs2) @@ -794,9 +802,7 @@ def has_pack_quantized_config(config): ) model_kwargs["max_memory"] = max_memory - model_kwargs2 = model_kwargs.copy() - if is_decilm: - model_kwargs2.pop("dtype", None) + model_kwargs2 = _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm) model = auto_model_module.from_pretrained( ckpt_path, device_map=device_map, diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index ec3c6a31a5c..00621ec6125 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -278,7 +278,7 @@ def from_config(config, **kwargs): def from_pretrained(*args, **kwargs): calls["from_pretrained"] = kwargs assert "dtype" not in kwargs - assert "torch_dtype" not in kwargs + assert kwargs["torch_dtype"] is torch.float16 return FakeModel() class FakeLlamaForCausalLM(FakeAutoModelForCausalLM): From 35d7dc59ffb62a919b4933621681025bf370535b Mon Sep 17 00:00:00 2001 From: realAsma Date: Tue, 30 Jun 2026 20:21:19 +0000 Subject: [PATCH 2/4] Rename HF PTQ dtype helper Signed-off-by: realAsma --- examples/hf_ptq/example_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 1f166de8e97..df0d4a5539f 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -626,7 +626,7 @@ def _get_config_dtype(config): return config_dtype -def _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm): +def _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm): model_kwargs = model_kwargs.copy() if is_decilm: model_kwargs["torch_dtype"] = config_dtype @@ -776,7 +776,7 @@ def has_pack_quantized_config(config): # When computing the device_map, assuming bfloat16 precision by default, # unless specified by the hf_config. config_dtype = _get_config_dtype(config_for_init) - model_kwargs2 = _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm) + model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm) if auto_model_module not in [AutoModelForCausalLM, AutoModel]: model_kwargs2.pop("trust_remote_code", None) if not is_decilm: @@ -802,7 +802,7 @@ def has_pack_quantized_config(config): ) model_kwargs["max_memory"] = max_memory - model_kwargs2 = _apply_decilm_config_dtype(model_kwargs, config_dtype, is_decilm) + model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm) model = auto_model_module.from_pretrained( ckpt_path, device_map=device_map, From 3ce217b3be7c070c7a013b742ffbce59b3204b7e Mon Sep 17 00:00:00 2001 From: realAsma Date: Tue, 30 Jun 2026 21:37:56 +0000 Subject: [PATCH 3/4] Move HF PTQ config dtype into helper Signed-off-by: realAsma --- examples/hf_ptq/example_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index df0d4a5539f..bab3eddcfe0 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -626,11 +626,13 @@ def _get_config_dtype(config): return config_dtype -def _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm): +def _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm, apply_config_dtype=False): model_kwargs = model_kwargs.copy() if is_decilm: model_kwargs["torch_dtype"] = config_dtype model_kwargs.pop("dtype", None) + elif apply_config_dtype: + model_kwargs["dtype"] = config_dtype return model_kwargs @@ -776,11 +778,11 @@ def has_pack_quantized_config(config): # When computing the device_map, assuming bfloat16 precision by default, # unless specified by the hf_config. config_dtype = _get_config_dtype(config_for_init) - model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm) + model_kwargs2 = _apply_dtype_to_config( + model_kwargs, config_dtype, is_decilm, apply_config_dtype=True + ) if auto_model_module not in [AutoModelForCausalLM, AutoModel]: model_kwargs2.pop("trust_remote_code", None) - if not is_decilm: - model_kwargs2["dtype"] = config_dtype model_kwargs2.pop("max_memory", None) model = from_config(config_for_init, **model_kwargs2) From 43ab80f33cf2232e3ec19212853dbd479fcd1052 Mon Sep 17 00:00:00 2001 From: realAsma Date: Wed, 1 Jul 2026 03:11:56 +0000 Subject: [PATCH 4/4] Infer DeciLM dtype handling in helper Signed-off-by: realAsma --- examples/hf_ptq/example_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index bab3eddcfe0..9e8dea5f107 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -626,9 +626,9 @@ def _get_config_dtype(config): return config_dtype -def _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm, apply_config_dtype=False): +def _apply_dtype_to_config(model_kwargs, config_dtype, architecture, apply_config_dtype=False): model_kwargs = model_kwargs.copy() - if is_decilm: + if "DeciLM" in architecture: model_kwargs["torch_dtype"] = config_dtype model_kwargs.pop("dtype", None) elif apply_config_dtype: @@ -769,7 +769,6 @@ def has_pack_quantized_config(config): auto_model_module = getattr(transformers, architecture) from_config = auto_model_module._from_config - is_decilm = "DeciLM" in architecture config_for_init = _resolve_init_config( hf_config, auto_model_module, ckpt_path, config_kwargs ) @@ -779,7 +778,7 @@ def has_pack_quantized_config(config): # unless specified by the hf_config. config_dtype = _get_config_dtype(config_for_init) model_kwargs2 = _apply_dtype_to_config( - model_kwargs, config_dtype, is_decilm, apply_config_dtype=True + model_kwargs, config_dtype, architecture, apply_config_dtype=True ) if auto_model_module not in [AutoModelForCausalLM, AutoModel]: model_kwargs2.pop("trust_remote_code", None) @@ -804,7 +803,7 @@ def has_pack_quantized_config(config): ) model_kwargs["max_memory"] = max_memory - model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, is_decilm) + model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, architecture) model = auto_model_module.from_pretrained( ckpt_path, device_map=device_map,