From c3097a73d2d3cfd15e07da30d2496024dd9184e8 Mon Sep 17 00:00:00 2001 From: Fridah-nv <201670829+Fridah-nv@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:45:59 +0000 Subject: [PATCH 1/2] Fix Nemotron-H PTQ AttributeError on Transformers 5.x with trust_remote_code Bug: Quantizing nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 via hf_ptq.py with --trust_remote_code fails on Transformers 5.x with "AttributeError: 'NemotronHConfig' object has no attribute 'moe_latent_size'" (works on 4.57.x). In example_utils.get_model, AutoConfig.from_pretrained(..., trust_remote_code=True) loads the checkpoint's bundled *remote* NemotronHConfig (authored for Transformers 4.55.4, no moe_latent_size). But since NemotronHForCausalLM is a *built-in* class in Transformers 5.x, the empty-weights device-map build uses the built-in model class, whose modeling code reads config.moe_latent_size -> the remote (old) config and the built-in (new) model are a mismatched pair. Transformers 4.57.x only worked because its built-in model never accessed that field. Fix: When instantiating the built-in model class, feed it a config from the same version as the model definition: if the loaded config came from remote code (class module under "transformers_modules"), re-derive it with the built-in class (AutoConfig without trust_remote_code) so required fields get their defaults. Non-remote configs are untouched. The real model load already resolves config via the built-in config_class, so only the device-map build needed aligning. Validated end-to-end: Nemotron-3-Nano-30B nvfp4 PTQ + export succeeds on Transformers 5.7.0; llm_ptq example unit tests pass. Signed-off-by: Fridah-nv <201670829+Fridah-nv@users.noreply.github.com> --- examples/hf_ptq/example_utils.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index c7a4d7a3b9a..fefbf8e2037 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -731,16 +731,27 @@ def has_pack_quantized_config(config): auto_model_module = getattr(transformers, architecture) from_config = auto_model_module._from_config + # Keep the config from the same version as the model definition: re-derive a + # remote-code config with the built-in class to avoid attribute mismatches. + config_for_init = hf_config + if auto_model_module not in [AutoModelForCausalLM, AutoModel] and type( + hf_config + ).__module__.startswith("transformers_modules"): + builtin_config_kwargs = { + k: v for k, v in config_kwargs.items() if k != "trust_remote_code" + } + config_for_init = AutoConfig.from_pretrained(ckpt_path, **builtin_config_kwargs) + with init_empty_weights(include_buffers=True): # When computing the device_map, assuming bfloat16 precision by default, # unless specified by the hf_config. - torch_dtype = getattr(hf_config, "torch_dtype", torch.bfloat16) + torch_dtype = getattr(config_for_init, "torch_dtype", torch.bfloat16) model_kwargs2 = model_kwargs.copy() if auto_model_module not in [AutoModelForCausalLM, AutoModel]: model_kwargs2.pop("trust_remote_code", None) model_kwargs2["dtype"] = torch_dtype model_kwargs2.pop("max_memory", None) - model = from_config(hf_config, **model_kwargs2) + model = from_config(config_for_init, **model_kwargs2) max_memory = get_max_memory() inferred_device_map = infer_auto_device_map(model, max_memory=max_memory) From 55250489f646d12042b91de648bd657d588d56b1 Mon Sep 17 00:00:00 2001 From: Fridah-nv <201670829+Fridah-nv@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:14:52 +0000 Subject: [PATCH 2/2] Address PR #1839 review: extract/harden config re-derivation + add unit tests - Extract the remote-code vs built-in config selection into _resolve_init_config() in example_utils.get_model, and harden it: wrap the built-in AutoConfig reload in try/except so an edge-case checkpoint (built-in architecture name but non-built-in model_type) falls back to hf_config instead of hard-crashing. - Add focused unit tests in tests/examples/llm_ptq/test_example_utils.py for the decision logic (re-derive for remote config, keep non-remote config, fall back when the reload raises), mocking AutoConfig.from_pretrained. Addresses reviewer feedback on missing tests and the unguarded reload. Signed-off-by: Fridah-nv <201670829+Fridah-nv@users.noreply.github.com> --- examples/hf_ptq/example_utils.py | 32 +++++++++++++------ tests/examples/hf_ptq/test_example_utils.py | 35 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index fefbf8e2037..1a8fd5703fe 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -598,6 +598,25 @@ def get_original_hf_quant_method(config) -> str | None: return None +def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs): + """Re-derive a built-in config when a remote-code config is used with a built-in model + class, so it matches the model definition's version; fall back to hf_config otherwise. + """ + if auto_model_module in [AutoModelForCausalLM, AutoModel]: + return hf_config + if not type(hf_config).__module__.startswith("transformers_modules"): + return hf_config + builtin_config_kwargs = {k: v for k, v in config_kwargs.items() if k != "trust_remote_code"} + try: + return AutoConfig.from_pretrained(ckpt_path, **builtin_config_kwargs) + except Exception as e: + warnings.warn( + f"Could not re-derive a built-in config for {ckpt_path} ({e}); using the " + "remote-code config for device-map inference." + ) + return hf_config + + def get_model( ckpt_path, device="cuda", @@ -731,16 +750,9 @@ def has_pack_quantized_config(config): auto_model_module = getattr(transformers, architecture) from_config = auto_model_module._from_config - # Keep the config from the same version as the model definition: re-derive a - # remote-code config with the built-in class to avoid attribute mismatches. - config_for_init = hf_config - if auto_model_module not in [AutoModelForCausalLM, AutoModel] and type( - hf_config - ).__module__.startswith("transformers_modules"): - builtin_config_kwargs = { - k: v for k, v in config_kwargs.items() if k != "trust_remote_code" - } - config_for_init = AutoConfig.from_pretrained(ckpt_path, **builtin_config_kwargs) + config_for_init = _resolve_init_config( + hf_config, auto_model_module, ckpt_path, config_kwargs + ) with init_empty_weights(include_buffers=True): # When computing the device_map, assuming bfloat16 precision by default, diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index d25da6e0ab2..e054f8a16d3 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -20,6 +20,7 @@ import json from types import SimpleNamespace +from unittest.mock import patch import torch from _test_utils.examples.hf_ptq_example_utils import example_utils @@ -194,3 +195,37 @@ def test_get_original_hf_quant_method_none_for_unquantized(): example_utils.get_original_hf_quant_method(SimpleNamespace(quantization_config=None)) is None ) + + +# ---------- _resolve_init_config --------------------------------------------- + + +def _remote_config(): + # Config whose class module lives under "transformers_modules" (remote code). + cls = type("_RemoteConfig", (), {"__module__": "transformers_modules.ckpt.config"}) + return cls() + + +def test_resolve_init_config_rederives_for_remote_config(): + builtin_cfg = SimpleNamespace() + with patch.object( + example_utils.AutoConfig, "from_pretrained", return_value=builtin_cfg + ) as mock: + out = example_utils._resolve_init_config( + _remote_config(), object, "/ckpt", {"trust_remote_code": True} + ) + assert out is builtin_cfg + mock.assert_called_once_with("/ckpt") # trust_remote_code stripped + + +def test_resolve_init_config_keeps_non_remote_config(): + cfg = SimpleNamespace() # module is "types", not remote + with patch.object(example_utils.AutoConfig, "from_pretrained") as mock: + assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg + mock.assert_not_called() + + +def test_resolve_init_config_falls_back_when_rederive_raises(): + cfg = _remote_config() + with patch.object(example_utils.AutoConfig, "from_pretrained", side_effect=ValueError()): + assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg