From 3c6b3948fb57dd93d7e55c85fc0c38cc7bc7c148 Mon Sep 17 00:00:00 2001
From: realAsma <akuriparambi@nvidia.com>
Date: Wed, 1 Jul 2026 20:47:29 +0000
Subject: [PATCH 1/5] Add TensorQuantizer rotate-back mode

Signed-off-by: realAsma <akuriparambi@nvidia.com>
---
 modelopt/torch/quantization/config.py         |  9 ++-
 .../nn/modules/tensor_quantizer.py            | 28 +++++--
 .../quantization/test_tensor_quant_cpu.py     | 78 ++++++++++++++++++-
 3 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/modelopt/torch/quantization/config.py b/modelopt/torch/quantization/config.py
index 344292264fa..d5629f7faaf 100644
--- a/modelopt/torch/quantization/config.py
+++ b/modelopt/torch/quantization/config.py
@@ -287,6 +287,7 @@ class RotateConfig(ModeloptBaseConfig):
     """
 
     enable: bool = False
+    mode: Literal["rotate", "rotate_back"] = "rotate"
     rotate_fp32: bool = False
     block_size: int | None = None
 
@@ -328,7 +329,7 @@ class QuantizerAttributeConfig(ModeloptBaseConfig):
     def validate_config(cls, values):
         """Validate quantizer config."""
 
-        def _validate_recursive(value):
+        def _validate_recursive(value, field_name=None):
             """Recursively validate config structure."""
             if value is None:
                 return
@@ -337,14 +338,16 @@ def _validate_recursive(value):
                 for item in value:
                     _validate_recursive(item)
             elif isinstance(value, dict):
+                if field_name == "rotate":
+                    return
                 if len(value) == 1 and "enable" in value and value["enable"] is True:
                     raise ValueError(
                         "Invalid quantizer config: Cannot specify only {'enable': True}. "
                         "Additional parameters are required when enabling quantization."
                     )
                 # Recurse into nested dicts
-                for v in value.values():
-                    _validate_recursive(v)
+                for k, v in value.items():
+                    _validate_recursive(v, k)
 
         _validate_recursive(values)
         return values
diff --git a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
index c50804dd2a9..7a47e769aea 100644
--- a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
+++ b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
@@ -619,6 +619,22 @@ def rotate_block_size(self):
             return self._rotate.get("block_size", None)
         return None
 
+    @property
+    def rotate_back_is_enabled(self):
+        """Check if inverse rotation should be applied after quantization."""
+        if isinstance(self._rotate, RotateConfig):
+            return self._rotate.enable and self._rotate.mode == "rotate_back"
+        if isinstance(self._rotate, dict) and self.rotate_is_enabled:
+            return self._rotate.get("mode", "rotate") == "rotate_back"
+        return False
+
+    def _rotate_inputs(self, inputs):
+        return normalized_hadamard_transform(
+            inputs,
+            rotate_fp32=self.rotate_is_fp32,
+            block_size=self.rotate_block_size,
+        )
+
     def disable_calib(self):
         """Disable calibration."""
         self._if_calib = False
@@ -1092,11 +1108,7 @@ def forward(self, inputs):
 
         # Rotating the input
         if self.rotate_is_enabled:
-            inputs = normalized_hadamard_transform(
-                inputs,
-                rotate_fp32=self.rotate_is_fp32,
-                block_size=self.rotate_block_size,
-            )
+            inputs = self._rotate_inputs(inputs)
 
         if self._disabled:
             # if quantizer is disabled, we still need to track the input dtype for saving the model
@@ -1149,6 +1161,8 @@ def forward(self, inputs):
                 with same_device_as(inputs):
                     outputs = self._fake_quantize(inputs)
             elif not self._dequantize:
+                if self.rotate_back_is_enabled:
+                    raise ValueError("rotate_back mode is only supported with fake_quant=True.")
                 outputs = self._real_quantize(inputs)
             else:
                 raise ValueError(
@@ -1159,6 +1173,9 @@ def forward(self, inputs):
         if self.is_static_block_quant:
             outputs = self._reset_to_original_shape(outputs)
 
+        if self.rotate_back_is_enabled and isinstance(outputs, torch.Tensor):
+            outputs = self._rotate_inputs(outputs)
+
         return outputs
 
     def _short_amax(self, fmt=".2e"):
@@ -1209,6 +1226,7 @@ def extra_repr(self):
             else ""
         )
         s += " rotated" if self.rotate_is_enabled else ""
+        s += " (rotate_back)" if self.rotate_back_is_enabled else ""
         s += " (fp32)" if self.rotate_is_fp32 else ""
         if self.rotate_block_size is not None:
             s += f" (block={self.rotate_block_size})"
diff --git a/tests/unit/torch/quantization/test_tensor_quant_cpu.py b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
index 218acd76f5d..b242aa3d0d2 100644
--- a/tests/unit/torch/quantization/test_tensor_quant_cpu.py
+++ b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
@@ -22,8 +22,13 @@
 from _test_utils.torch.quantization.tensor_quant_common import FakeTensorQuantTester
 
 import modelopt.torch.quantization as mtq
+import modelopt.torch.quantization.nn.modules.tensor_quantizer as tensor_quantizer_module
 from modelopt.torch.quantization.config import QuantizerAttributeConfig
-from modelopt.torch.quantization.nn import TensorQuantizer
+from modelopt.torch.quantization.nn import (
+    TensorQuantizer,
+    register_quant_backend,
+    unregister_quant_backend,
+)
 
 
 class TestFakeTensorQuantCPU(FakeTensorQuantTester):
@@ -56,6 +61,18 @@ def test_from_to_dict(self, verbose):
         quant_attr_cfg_2 = QuantizerAttributeConfig(**quant_attr_cfg_1.dict())
         assert quant_attr_cfg_1 == quant_attr_cfg_2
 
+    def test_rotate_mode_serialization(self):
+        quant_attr_cfg = QuantizerAttributeConfig(
+            rotate={"enable": True, "mode": "rotate_back", "rotate_fp32": True, "block_size": 8}
+        )
+
+        assert quant_attr_cfg.model_dump(exclude_unset=True)["rotate"] == {
+            "enable": True,
+            "mode": "rotate_back",
+            "rotate_fp32": True,
+            "block_size": 8,
+        }
+
     def test_num_bits(self):
         """Test num_bits for both integer and tuple cases."""
 
@@ -92,6 +109,65 @@ def test_num_bits(self):
             QuantizerAttributeConfig(enable=True, num_bits=(-1, 2))
 
 
+def _run_rotated_backend(monkeypatch, rotate):
+    calls = []
+
+    def rotate_fn(inputs, rotate_fp32=False, block_size=None):
+        calls.append((rotate_fp32, block_size))
+        return inputs + 10
+
+    def backend(inputs, _tq):
+        return inputs * 2
+
+    monkeypatch.setattr(tensor_quantizer_module, "normalized_hadamard_transform", rotate_fn)
+    register_quant_backend("test_rotate_mode_backend", backend)
+    try:
+        quantizer = TensorQuantizer(
+            QuantizerAttributeConfig(rotate=rotate, backend="test_rotate_mode_backend")
+        )
+        inputs = torch.tensor([[1.0, 2.0]])
+        return quantizer(inputs), inputs, calls, quantizer
+    finally:
+        unregister_quant_backend("test_rotate_mode_backend")
+
+
+def test_tensor_quantizer_rotate_mode_preserves_default_path(monkeypatch):
+    outputs, inputs, calls, quantizer = _run_rotated_backend(monkeypatch, rotate={"enable": True})
+
+    assert not quantizer.rotate_back_is_enabled
+    assert torch.equal(outputs, (inputs + 10) * 2)
+    assert calls == [(False, None)]
+
+
+def test_tensor_quantizer_rotate_mode_can_rotate_back(monkeypatch):
+    outputs, inputs, calls, quantizer = _run_rotated_backend(
+        monkeypatch,
+        rotate={"enable": True, "mode": "rotate_back", "rotate_fp32": True, "block_size": 8},
+    )
+
+    assert quantizer.rotate_back_is_enabled
+    assert torch.equal(outputs, ((inputs + 10) * 2) + 10)
+    assert calls == [(True, 8), (True, 8)]
+
+
+def test_tensor_quantizer_rotate_back_rejects_real_quant(monkeypatch):
+    monkeypatch.setattr(
+        tensor_quantizer_module,
+        "normalized_hadamard_transform",
+        lambda inputs, rotate_fp32=False, block_size=None: inputs,
+    )
+    quantizer = TensorQuantizer(
+        QuantizerAttributeConfig(
+            num_bits=8,
+            fake_quant=False,
+            rotate={"enable": True, "mode": "rotate_back"},
+        )
+    )
+
+    with pytest.raises(ValueError, match="rotate_back mode is only supported with fake_quant=True"):
+        quantizer(torch.tensor([[1.0, 2.0]]))
+
+
 WINT4INT8_CFG = {
     "quant_cfg": [
         {"quantizer_name": "*", "enable": False},

From 4b2d0e6915935021e4f775fa4999be4e066ed9c7 Mon Sep 17 00:00:00 2001
From: realAsma <akuriparambi@nvidia.com>
Date: Wed, 1 Jul 2026 23:33:22 +0000
Subject: [PATCH 2/5] Add Hadamard roundtrip GPU test

Signed-off-by: realAsma <akuriparambi@nvidia.com>
---
 tests/gpu/torch/quantization/test_hadamard.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/gpu/torch/quantization/test_hadamard.py b/tests/gpu/torch/quantization/test_hadamard.py
index 1173eba2097..7034feda75c 100644
--- a/tests/gpu/torch/quantization/test_hadamard.py
+++ b/tests/gpu/torch/quantization/test_hadamard.py
@@ -48,6 +48,9 @@ def test_hadamard_transform(dim):
     xxt_h = x_h @ x_h.T
     # The numerical error can be large, especially for 16-bit floats.
     assert torch.allclose(xxt_h, xxt, atol=0.05)
+    x_roundtrip = normalized_hadamard_transform(x_h)
+    assert torch.allclose(x_roundtrip, x, rtol=1e-5, atol=1e-6)
+
     x_h_fp32 = normalized_hadamard_transform(x, rotate_fp32=True)
     xxt_h_fp32 = x_h_fp32 @ x_h_fp32.T
     assert torch.allclose(xxt_h_fp32, xxt, atol=0.05)
@@ -66,6 +69,8 @@ def test_hadamard_transform_block(dim, block_size):
     # Use rtol instead of atol: float32 accumulated error scales with value magnitude,
     # which grows with dim. 1e-3 relative tolerance is appropriate for float32 block RHT.
     assert torch.allclose(xxt_h, xxt, rtol=1e-3, atol=1e-6)
+    x_roundtrip = normalized_hadamard_transform(x_h, block_size=block_size)
+    assert torch.allclose(x_roundtrip, x, rtol=1e-5, atol=1e-6)
 
 
 @pytest.mark.parametrize(

From afcbf784f7530338b2c39058073099fc7cdb9249 Mon Sep 17 00:00:00 2001
From: realAsma <akuriparambi@nvidia.com>
Date: Thu, 2 Jul 2026 00:23:35 +0000
Subject: [PATCH 3/5] Address rotate-back review feedback

Signed-off-by: realAsma <akuriparambi@nvidia.com>
---
 CHANGELOG.rst                                              | 1 +
 modelopt/torch/quantization/nn/modules/tensor_quantizer.py | 5 +++--
 tests/unit/torch/quantization/test_tensor_quant_cpu.py     | 5 ++++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 7960dac388b..daa7a66c1f5 100755
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -17,6 +17,7 @@ Changelog
 
 **New Features**
 
+- Add ``rotate.mode`` to torch quantizer configs. The default ``"rotate"`` keeps the existing rotate-before-quantize behavior; ``"rotate_back"`` enables fake-quant rotate → quantize → rotate-back for TensorQuantizer.
 - Add the **D-PACE** loss objective for DFlash speculative-decoding training (`arXiv:2605.18810 <https://arxiv.org/abs/2605.18810>`_) and make it the default (``dflash_loss_objective: dpace``). It replaces the static exponential position decay with dynamic, confidence-derived per-position weights that adapt to whichever block positions currently limit acceptance. Smoothing is controlled by ``dflash_dpace_alpha`` (default 0.5); set ``dflash_loss_objective: decay`` to restore the previous static schedule. Training-only and detached from the gradient (no architecture or inference change).
 - Add the ``day0-release`` agent skill (``.agents/skills/day0-release/``), a deterministic end-to-end driver that chains the PTQ → evaluation → comparison skills (the evaluation stage deploys the checkpoint itself) with an enforced gate after each stage and returns a publish decision (ACCEPT / REGRESSION / ANOMALOUS / INFEASIBLE). Ships three GPU-free, unit-tested gate scripts (``gate_ptq.py``, ``gate_run.py``, ``gate_compare.py``) that validate checkpoint coverage, evaluation-run completeness, and baseline-vs-candidate accuracy threshold. v1 reports and stops on regression; the recipe-search loop is deferred.
 - Add **streaming** speculative-decoding training (EAGLE3 / DFlash): the draft trains on base-model hidden states produced on the fly by a co-located ``vllm serve`` (no disk dump), moved trainer-side over NIXL RDMA, scaling to multi-node (dedicated serve replicas + DDP trainers). New launcher examples for NVFP4 Kimi-K2.5 / K2.6 on GB200/aarch64 under ``tools/launcher/examples/moonshotai/``.
diff --git a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
index 7a47e769aea..755dce7a45c 100644
--- a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
+++ b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
@@ -1106,6 +1106,9 @@ def forward(self, inputs):
         if self.pre_quant_scale is not None:
             inputs = inputs * self.pre_quant_scale
 
+        if self.rotate_back_is_enabled and self._if_quant and not self.fake_quant:
+            raise ValueError("rotate_back mode is only supported with fake_quant=True.")
+
         # Rotating the input
         if self.rotate_is_enabled:
             inputs = self._rotate_inputs(inputs)
@@ -1161,8 +1164,6 @@ def forward(self, inputs):
                 with same_device_as(inputs):
                     outputs = self._fake_quantize(inputs)
             elif not self._dequantize:
-                if self.rotate_back_is_enabled:
-                    raise ValueError("rotate_back mode is only supported with fake_quant=True.")
                 outputs = self._real_quantize(inputs)
             else:
                 raise ValueError(
diff --git a/tests/unit/torch/quantization/test_tensor_quant_cpu.py b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
index b242aa3d0d2..139ca232b1b 100644
--- a/tests/unit/torch/quantization/test_tensor_quant_cpu.py
+++ b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
@@ -151,10 +151,13 @@ def test_tensor_quantizer_rotate_mode_can_rotate_back(monkeypatch):
 
 
 def test_tensor_quantizer_rotate_back_rejects_real_quant(monkeypatch):
+    def fail_if_rotated(inputs, rotate_fp32=False, block_size=None):
+        raise AssertionError("rotate_back with fake_quant=False should fail before rotation")
+
     monkeypatch.setattr(
         tensor_quantizer_module,
         "normalized_hadamard_transform",
-        lambda inputs, rotate_fp32=False, block_size=None: inputs,
+        fail_if_rotated,
     )
     quantizer = TensorQuantizer(
         QuantizerAttributeConfig(

From fe810d4c1040da4f9afffe8596739e8cdee98128 Mon Sep 17 00:00:00 2001
From: realAsma <akuriparambi@nvidia.com>
Date: Thu, 2 Jul 2026 00:47:01 +0000
Subject: [PATCH 4/5] Add TensorQuantizer random Hadamard rotation seed

Signed-off-by: realAsma <akuriparambi@nvidia.com>
---
 CHANGELOG.rst                                 |  1 +
 modelopt/torch/quantization/config.py         |  9 ++++++
 modelopt/torch/quantization/nn/functional.py  | 23 ++++++++++++-
 .../nn/modules/tensor_quantizer.py            | 17 ++++++++--
 tests/gpu/torch/quantization/test_hadamard.py | 14 ++++++++
 .../quantization/test_tensor_quant_cpu.py     | 32 +++++++++++++++----
 6 files changed, 86 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index daa7a66c1f5..3bdb6e70698 100755
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -18,6 +18,7 @@ Changelog
 **New Features**
 
 - Add ``rotate.mode`` to torch quantizer configs. The default ``"rotate"`` keeps the existing rotate-before-quantize behavior; ``"rotate_back"`` enables fake-quant rotate → quantize → rotate-back for TensorQuantizer.
+- Add ``rotate.seed`` to torch quantizer configs. The default ``None`` keeps regular Hadamard rotation; an integer seed enables deterministic Random Hadamard Transform signs.
 - Add the **D-PACE** loss objective for DFlash speculative-decoding training (`arXiv:2605.18810 <https://arxiv.org/abs/2605.18810>`_) and make it the default (``dflash_loss_objective: dpace``). It replaces the static exponential position decay with dynamic, confidence-derived per-position weights that adapt to whichever block positions currently limit acceptance. Smoothing is controlled by ``dflash_dpace_alpha`` (default 0.5); set ``dflash_loss_objective: decay`` to restore the previous static schedule. Training-only and detached from the gradient (no architecture or inference change).
 - Add the ``day0-release`` agent skill (``.agents/skills/day0-release/``), a deterministic end-to-end driver that chains the PTQ → evaluation → comparison skills (the evaluation stage deploys the checkpoint itself) with an enforced gate after each stage and returns a publish decision (ACCEPT / REGRESSION / ANOMALOUS / INFEASIBLE). Ships three GPU-free, unit-tested gate scripts (``gate_ptq.py``, ``gate_run.py``, ``gate_compare.py``) that validate checkpoint coverage, evaluation-run completeness, and baseline-vs-candidate accuracy threshold. v1 reports and stops on regression; the recipe-search loop is deferred.
 - Add **streaming** speculative-decoding training (EAGLE3 / DFlash): the draft trains on base-model hidden states produced on the fly by a co-located ``vllm serve`` (no disk dump), moved trainer-side over NIXL RDMA, scaling to multi-node (dedicated serve replicas + DDP trainers). New launcher examples for NVFP4 Kimi-K2.5 / K2.6 on GB200/aarch64 under ``tools/launcher/examples/moonshotai/``.
diff --git a/modelopt/torch/quantization/config.py b/modelopt/torch/quantization/config.py
index d5629f7faaf..4618a0b3e8a 100644
--- a/modelopt/torch/quantization/config.py
+++ b/modelopt/torch/quantization/config.py
@@ -290,6 +290,7 @@ class RotateConfig(ModeloptBaseConfig):
     mode: Literal["rotate", "rotate_back"] = "rotate"
     rotate_fp32: bool = False
     block_size: int | None = None
+    seed: int | None = None
 
     @field_validator("block_size", mode="before")
     @classmethod
@@ -299,6 +300,14 @@ def validate_block_size(cls, v):
             raise ValueError(f"block_size must be a positive int, got {v!r}")
         return v
 
+    @field_validator("seed", mode="before")
+    @classmethod
+    def validate_seed(cls, v):
+        """Validate seed is a non-negative int (mode=before to catch bool before int coercion)."""
+        if v is not None and (isinstance(v, bool) or not isinstance(v, int) or v < 0):
+            raise ValueError(f"seed must be a non-negative int, got {v!r}")
+        return v
+
 
 class QuantizerAttributeConfig(ModeloptBaseConfig):
     """Quantizer attribute type."""
diff --git a/modelopt/torch/quantization/nn/functional.py b/modelopt/torch/quantization/nn/functional.py
index 94cbcf74fda..25b9992ef23 100644
--- a/modelopt/torch/quantization/nn/functional.py
+++ b/modelopt/torch/quantization/nn/functional.py
@@ -98,7 +98,15 @@ def _largest_pow2_divisor(n: int) -> int:
     return n & (-n)
 
 
-def normalized_hadamard_transform(inputs, rotate_fp32=False, block_size=None):
+def _random_signs(seed: int, dim: int, device: torch.device, dtype: torch.dtype):
+    generator = torch.Generator(device=device).manual_seed(seed)
+    signs = torch.randint(0, 2, (dim,), generator=generator, device=device, dtype=torch.int8)
+    return signs.to(dtype=dtype).mul_(2).sub_(1)
+
+
+def normalized_hadamard_transform(
+    inputs, rotate_fp32=False, block_size=None, random_sign_seed=None, inverse=False
+):
     """Normalized fast hadamard transform.
 
     Supports block-granular RHT for dimensions that are not a power of 2.
@@ -112,6 +120,9 @@ def normalized_hadamard_transform(inputs, rotate_fp32=False, block_size=None):
         block_size: Block size for block-granular RHT. Must be power of 2 and divide
             inputs.shape[-1]. If None: use full-dimension FHT when dim is power of 2;
             otherwise auto-select the largest power-of-2 divisor of the dimension.
+        random_sign_seed: If set, apply a deterministic random sign diagonal before
+            Hadamard. The same seed is used for the inverse transform.
+        inverse: If True, apply inverse RHT order for seeded transforms.
 
     Returns:
         Rotated tensor with same shape as inputs.
@@ -129,6 +140,13 @@ def normalized_hadamard_transform(inputs, rotate_fp32=False, block_size=None):
     dtype = inputs.dtype
     if rotate_fp32:
         inputs = inputs.to(torch.float32)
+    signs = (
+        _random_signs(random_sign_seed, dim, inputs.device, inputs.dtype)
+        if random_sign_seed is not None
+        else None
+    )
+    if signs is not None and not inverse:
+        inputs = inputs * signs
 
     if block_size is None and utils.is_pow2(dim):
         # Full-dimension FHT (original behavior)
@@ -161,4 +179,7 @@ def normalized_hadamard_transform(inputs, rotate_fp32=False, block_size=None):
         )
         outputs = rotated.reshape(inputs.shape)
 
+    if signs is not None and inverse:
+        outputs = outputs * signs
+
     return outputs.to(dtype) if rotate_fp32 else outputs
diff --git a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
index 755dce7a45c..60a7b9ee6f5 100644
--- a/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
+++ b/modelopt/torch/quantization/nn/modules/tensor_quantizer.py
@@ -619,6 +619,15 @@ def rotate_block_size(self):
             return self._rotate.get("block_size", None)
         return None
 
+    @property
+    def rotate_seed(self):
+        """Seed for deterministic RHT signs, or None for normalized HT."""
+        if isinstance(self._rotate, RotateConfig):
+            return self._rotate.seed if self._rotate.enable else None
+        if isinstance(self._rotate, dict) and self.rotate_is_enabled:
+            return self._rotate.get("seed", None)
+        return None
+
     @property
     def rotate_back_is_enabled(self):
         """Check if inverse rotation should be applied after quantization."""
@@ -628,11 +637,13 @@ def rotate_back_is_enabled(self):
             return self._rotate.get("mode", "rotate") == "rotate_back"
         return False
 
-    def _rotate_inputs(self, inputs):
+    def _rotate_inputs(self, inputs, inverse=False):
         return normalized_hadamard_transform(
             inputs,
             rotate_fp32=self.rotate_is_fp32,
             block_size=self.rotate_block_size,
+            random_sign_seed=self.rotate_seed,
+            inverse=inverse,
         )
 
     def disable_calib(self):
@@ -1175,7 +1186,7 @@ def forward(self, inputs):
             outputs = self._reset_to_original_shape(outputs)
 
         if self.rotate_back_is_enabled and isinstance(outputs, torch.Tensor):
-            outputs = self._rotate_inputs(outputs)
+            outputs = self._rotate_inputs(outputs, inverse=True)
 
         return outputs
 
@@ -1231,6 +1242,8 @@ def extra_repr(self):
         s += " (fp32)" if self.rotate_is_fp32 else ""
         if self.rotate_block_size is not None:
             s += f" (block={self.rotate_block_size})"
+        if self.rotate_seed is not None:
+            s += f" (seed={self.rotate_seed})"
         s += (
             f" calibrator={self._calibrator.__class__.__name__}"
             if (self._calibrator is not None)
diff --git a/tests/gpu/torch/quantization/test_hadamard.py b/tests/gpu/torch/quantization/test_hadamard.py
index 7034feda75c..7da6a3ca122 100644
--- a/tests/gpu/torch/quantization/test_hadamard.py
+++ b/tests/gpu/torch/quantization/test_hadamard.py
@@ -73,6 +73,20 @@ def test_hadamard_transform_block(dim, block_size):
     assert torch.allclose(x_roundtrip, x, rtol=1e-5, atol=1e-6)
 
 
+@pytest.mark.parametrize("block_size", [None, 4])
+def test_hadamard_transform_seeded_inverse(block_size):
+    x = torch.rand(4, 8, device="cuda")
+    x_h = normalized_hadamard_transform(x, block_size=block_size, random_sign_seed=123)
+    x_roundtrip = normalized_hadamard_transform(
+        x_h, block_size=block_size, random_sign_seed=123, inverse=True
+    )
+
+    assert torch.allclose(x_roundtrip, x, rtol=1e-5, atol=1e-6)
+    assert torch.allclose(
+        x_h, normalized_hadamard_transform(x, block_size=block_size, random_sign_seed=123)
+    )
+
+
 @pytest.mark.parametrize(
     "rotate_fp32",
     [True, False],
diff --git a/tests/unit/torch/quantization/test_tensor_quant_cpu.py b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
index 139ca232b1b..d3f6b59ce45 100644
--- a/tests/unit/torch/quantization/test_tensor_quant_cpu.py
+++ b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
@@ -63,7 +63,13 @@ def test_from_to_dict(self, verbose):
 
     def test_rotate_mode_serialization(self):
         quant_attr_cfg = QuantizerAttributeConfig(
-            rotate={"enable": True, "mode": "rotate_back", "rotate_fp32": True, "block_size": 8}
+            rotate={
+                "enable": True,
+                "mode": "rotate_back",
+                "rotate_fp32": True,
+                "block_size": 8,
+                "seed": 123,
+            }
         )
 
         assert quant_attr_cfg.model_dump(exclude_unset=True)["rotate"] == {
@@ -71,8 +77,12 @@ def test_rotate_mode_serialization(self):
             "mode": "rotate_back",
             "rotate_fp32": True,
             "block_size": 8,
+            "seed": 123,
         }
 
+        with pytest.raises(ValueError, match="seed must be a non-negative int"):
+            QuantizerAttributeConfig(rotate={"enable": True, "seed": -1})
+
     def test_num_bits(self):
         """Test num_bits for both integer and tuple cases."""
 
@@ -112,8 +122,8 @@ def test_num_bits(self):
 def _run_rotated_backend(monkeypatch, rotate):
     calls = []
 
-    def rotate_fn(inputs, rotate_fp32=False, block_size=None):
-        calls.append((rotate_fp32, block_size))
+    def rotate_fn(inputs, rotate_fp32=False, block_size=None, random_sign_seed=None, inverse=False):
+        calls.append((rotate_fp32, block_size, random_sign_seed, inverse))
         return inputs + 10
 
     def backend(inputs, _tq):
@@ -136,22 +146,30 @@ def test_tensor_quantizer_rotate_mode_preserves_default_path(monkeypatch):
 
     assert not quantizer.rotate_back_is_enabled
     assert torch.equal(outputs, (inputs + 10) * 2)
-    assert calls == [(False, None)]
+    assert calls == [(False, None, None, False)]
 
 
 def test_tensor_quantizer_rotate_mode_can_rotate_back(monkeypatch):
     outputs, inputs, calls, quantizer = _run_rotated_backend(
         monkeypatch,
-        rotate={"enable": True, "mode": "rotate_back", "rotate_fp32": True, "block_size": 8},
+        rotate={
+            "enable": True,
+            "mode": "rotate_back",
+            "rotate_fp32": True,
+            "block_size": 8,
+            "seed": 123,
+        },
     )
 
     assert quantizer.rotate_back_is_enabled
     assert torch.equal(outputs, ((inputs + 10) * 2) + 10)
-    assert calls == [(True, 8), (True, 8)]
+    assert calls == [(True, 8, 123, False), (True, 8, 123, True)]
 
 
 def test_tensor_quantizer_rotate_back_rejects_real_quant(monkeypatch):
-    def fail_if_rotated(inputs, rotate_fp32=False, block_size=None):
+    def fail_if_rotated(
+        inputs, rotate_fp32=False, block_size=None, random_sign_seed=None, inverse=False
+    ):
         raise AssertionError("rotate_back with fake_quant=False should fail before rotation")
 
     monkeypatch.setattr(

From 5f779094a8bec6c06444bce78dc9e337369d44ba Mon Sep 17 00:00:00 2001
From: realAsma <akuriparambi@nvidia.com>
Date: Thu, 2 Jul 2026 01:21:15 +0000
Subject: [PATCH 5/5] Cache Random Hadamard signs

Signed-off-by: realAsma <akuriparambi@nvidia.com>
---
 modelopt/torch/quantization/nn/functional.py  |  2 ++
 .../quantization/test_tensor_quant_cpu.py     | 21 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/modelopt/torch/quantization/nn/functional.py b/modelopt/torch/quantization/nn/functional.py
index 25b9992ef23..2a5f78f1817 100644
--- a/modelopt/torch/quantization/nn/functional.py
+++ b/modelopt/torch/quantization/nn/functional.py
@@ -16,6 +16,7 @@
 """Some supportive functions."""
 
 import warnings
+from functools import lru_cache
 
 import torch
 from torch.autograd import Function
@@ -98,6 +99,7 @@ def _largest_pow2_divisor(n: int) -> int:
     return n & (-n)
 
 
+@lru_cache(maxsize=16)
 def _random_signs(seed: int, dim: int, device: torch.device, dtype: torch.dtype):
     generator = torch.Generator(device=device).manual_seed(seed)
     signs = torch.randint(0, 2, (dim,), generator=generator, device=device, dtype=torch.int8)
diff --git a/tests/unit/torch/quantization/test_tensor_quant_cpu.py b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
index d3f6b59ce45..b21375ef8e4 100644
--- a/tests/unit/torch/quantization/test_tensor_quant_cpu.py
+++ b/tests/unit/torch/quantization/test_tensor_quant_cpu.py
@@ -29,6 +29,7 @@
     register_quant_backend,
     unregister_quant_backend,
 )
+from modelopt.torch.quantization.nn.functional import _random_signs
 
 
 class TestFakeTensorQuantCPU(FakeTensorQuantTester):
@@ -189,6 +190,26 @@ def fail_if_rotated(
         quantizer(torch.tensor([[1.0, 2.0]]))
 
 
+def test_random_signs_are_cached_by_seed_dim_device_and_dtype():
+    _random_signs.cache_clear()
+
+    try:
+        signs = _random_signs(123, 8, torch.device("cpu"), torch.float32)
+        cached_signs = _random_signs(123, 8, torch.device("cpu"), torch.float32)
+        fp16_signs = _random_signs(123, 8, torch.device("cpu"), torch.float16)
+        different_seed_signs = _random_signs(124, 8, torch.device("cpu"), torch.float32)
+        different_dim_signs = _random_signs(123, 16, torch.device("cpu"), torch.float32)
+
+        assert cached_signs.data_ptr() == signs.data_ptr()
+        assert torch.equal(cached_signs, signs)
+        assert fp16_signs.dtype == torch.float16
+        assert fp16_signs.data_ptr() != signs.data_ptr()
+        assert different_seed_signs.data_ptr() != signs.data_ptr()
+        assert different_dim_signs.data_ptr() != signs.data_ptr()
+    finally:
+        _random_signs.cache_clear()
+
+
 WINT4INT8_CFG = {
     "quant_cfg": [
         {"quantizer_name": "*", "enable": False},