Fix SmolLM3 inverted no_rope_layers logic and add L3/L5 tests (#233)

## Summary

Fix inverted no_rope_layers interpretation in SmolLM3, fix BF16 type
mismatch in RoPE, and add missing L3/L5 tests.

## Bug Fix: Inverted no_rope_layers

HuggingFace convention: no_rope_layers[i] == 1 means USE RoPE (despite
the misleading name). Mobius had it backwards, causing 27/36 layers to
have wrong RoPE gating.

## Bug Fix: BF16 RoPE Type Mismatch

The ONNX RotaryEmbedding op requires all tensor inputs (x, cos, sin) to
share the same type T. The RoPE cos/sin cache is always computed in FP32
(numpy limitation), so BF16/FP16 models had a type constraint violation:
BF16 query/key + FP32 cos/sin. This cascaded through Attention -> o_proj
-> residual Add causing ORT type errors.

Fix: Added a dtype parameter to BaseRope that all RoPE subclasses thread
from config.dtype. BaseRope._cast_embeddings() inserts Cast ops after
gathering cos/sin when the model dtype differs from FP32. For FP32
models, no Cast is added (zero overhead). Covers all RoPE variants.

Also fixes a latent bug in YarnRope where attn_scale dtype was derived
from cos_cache.dtype (always FP32), fixing Ministral3/Mistral4 BF16
support.

## Results

- Prefill max diff: 6.87 -> 0.00005 (143,000x improvement)
- L5 generation: now matches HF exactly (100% token match)
- BF16 model: zero Add type mismatches (verified with real SmolLM3-3B
weights)

## Tests

- L3: Added integration test entry for smollm3-3b
- L5: Removed xfail, generation now passes
- L1: Added no_rope_layers to tiny config for direct gating coverage
- All 1185 L1 tests + 1463 unit tests pass

---------

Signed-off-by: Justin Chu <justinchu@microsoft.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: justinchuby

src/mobius/models/smollm.py

@@ -78,14 +78,17 @@ def forward(
             attn_bias = (
                 sliding_attn_bias if layer_type == "sliding_attention" else full_attn_bias
             )
-            # SmolLM3 uses no_rope_layers to gate RoPE per layer:
-            # no_rope_layers[i] == 1 → skip RoPE, 0 → apply RoPE
-            skip_rope = (
-                self.no_rope_layers is not None
-                and i < len(self.no_rope_layers)
-                and self.no_rope_layers[i] == 1
+            # SmolLM3 uses no_rope_layers to gate RoPE per layer.
+            # Despite the name, the HF convention is:
+            #   no_rope_layers[i] == 1 → USE RoPE
+            #   no_rope_layers[i] == 0 → skip RoPE
+            # (HF assigns self.use_rope = config.no_rope_layers[layer_idx])
+            use_rope = (
+                self.no_rope_layers is None
+                or i >= len(self.no_rope_layers)
+                or self.no_rope_layers[i] == 1
             )
-            rope = None if skip_rope else position_embeddings
+            rope = position_embeddings if use_rope else None
 
             hidden_states, present_kv = layer(
                 op,

testdata/golden/causal-lm/smollm3-3b.json

@@ -14,22 +14,22 @@
     1131
   ],
   "top10_logits": [
-    "0x1.b324e60000000p+4",
-    "0x1.a30b7a0000000p+4",
-    "0x1.87b1fa0000000p+4",
-    "0x1.76de420000000p+4",
-    "0x1.6d041e0000000p+4",
-    "0x1.5c8ad40000000p+4",
-    "0x1.5144c40000000p+4",
-    "0x1.513a040000000p+4",
-    "0x1.5008a00000000p+4",
-    "0x1.4f57620000000p+4"
+    "0x1.b324e00000000p+4",
+    "0x1.a30b740000000p+4",
+    "0x1.87b1f00000000p+4",
+    "0x1.76de380000000p+4",
+    "0x1.6d04120000000p+4",
+    "0x1.5c8ad00000000p+4",
+    "0x1.5144c00000000p+4",
+    "0x1.5139fa0000000p+4",
+    "0x1.5008980000000p+4",
+    "0x1.4f575a0000000p+4"
   ],
   "logits_summary": [
-    "0x1.b324e60000000p+4",
-    "-0x1.d251fa0000000p+3",
-    "0x1.662fc1b5f12a7p+2",
-    "0x1.a02a75d17f04cp+1"
+    "0x1.b324e00000000p+4",
+    "-0x1.d2520c0000000p+3",
+    "0x1.662fad9be0416p+2",
+    "0x1.a02a76b63e452p+1"
   ],
   "input_ids": [
     12805,

tests/_test_configs.py

@@ -409,7 +409,7 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig:
         True,
     ),
     ("qwen3_vl_text", {"attn_qk_norm": True}, False),
-    ("smollm3", {}, False),
+    ("smollm3", {"no_rope_layers": [1, 0]}, True),  # exercise per-layer RoPE gating
     # === Mixture of Experts ===
     (
         "phimoe",

tests/e2e_golden_test.py

@@ -155,7 +155,6 @@ def _use_temp_hf_cache(tmp_path):
     "text-generation/helium-1-2b": "Helium decode loop diverges from HF after first token",
     "text-generation/nanochat-d20": "NanoChat decode loop diverges from HF after first token",
     "text-generation/ernie4_5-0_3b": "ERNIE 4.5 decode loop diverges from HF after first token",
-    "text-generation/smollm3-3b": "SmolLM3 3B decode loop diverges from HF (FP32 precision with 3B params)",
     # MLA compressed KV cache dimensions not yet handled by OnnxGenerator
     "text-generation/youtu-2b": "Youtu MLA KV cache dims differ from standard attention (v_head_dim != head_dim)",
 }

tests/integration_test.py

@@ -86,6 +86,8 @@ def _model_accessible(model_id: str) -> bool:
     # CausalLMModel (base: llama/mistral/qwen2 architecture)
     pytest.param("Qwen/Qwen2.5-0.5B", False, id="qwen2.5-0.5b"),
     pytest.param("HuggingFaceTB/SmolLM-135M", False, id="smollm-135m"),
+    # SmolLM3 (per-layer RoPE gating via no_rope_layers)
+    pytest.param("HuggingFaceTB/SmolLM3-3B", False, id="smollm3-3b"),
     # Gemma
     pytest.param("google/gemma-3-1b-pt", False, id="gemma3-1b"),
     # Granite