Emit MatMulNBits accuracy_level=4 (int8 path) — 2.3x CPU decode (#402)

**Root cause of onnx-genai being ~3.6x slower than llama.cpp on CPU:
MatMulNBits was emitted with no `accuracy_level`**, so ORT's MLAS kernel
ran the fp32 dequant+GEMM path instead of the int8 dynamic-quant + int8
dot-product path (ARM SDOT / x86 AVX-VNNI) — the same class of kernel
llama.cpp uses. ORT's parity claim holds; we just weren't opting in.

`default_int4_accuracy_level=4` already existed on the cpu/webgpu
`EpCapabilities` but was **dead config**. This plumbs it via a new
`_accuracy_level_attrs()` helper into both `MatMulNBits` emission sites
in `components/_quantized_linear.py` (Q/K/V/O, MLP, tied + non-tied
head). Emits when `>0`, omits at `0` (portable default preserved).

### Measured (Qwen2.5-0.5B Q4, CPU EP, decode tok/s, all coherent)
| config | tok/s |
|---|---:|
| baseline (accuracy_level missing) | 39.3 |
| **accuracy_level=4 (int8)** | **91.8** (2.33x) |
| quantized head + acc4 | **194.7** |
| ref: LM Studio CPU | 157 |

fp16/bf16 levels regress on M1 (no native GEMM). End-to-end verified:
`--ep cpu` stamps all 168 nodes with accuracy_level=4; `--ep default`
omits it.

lintrunner clean; pytest 518 passed (+2 tests).

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: justinchuby

src/mobius/components/_quantized_linear.py

@@ -15,6 +15,8 @@
 import onnx_ir as ir
 from onnxscript import OpBuilder, nn
 
+from mobius._build_context import ep_capabilities
+
 # MatMulNBits packs weights into uint8 blobs.  The packed shape depends
 # on bits and block_size:
 #   packed_weights: [N, n_blocks, blob_size]  (uint8)
@@ -26,6 +28,29 @@
 _MICROSOFT_DOMAIN = "com.microsoft"
 
 
+def _accuracy_level_attrs(bits: int) -> dict[str, int]:
+    """Return the ``accuracy_level`` attribute for ``MatMulNBits``, if any.
+
+    Only emitted for 4-bit weights: ``accuracy_level`` is sourced from
+    ``EpCapabilities.default_int4_accuracy_level`` and its int8-accumulation
+    semantics are defined for INT4 ``MatMulNBits``. For 2-bit / 8-bit weights the
+    attribute is omitted so those paths keep ORT's default behavior.
+
+    ORT's MLAS CPU ``MatMulNBits`` kernel selects its compute path from the
+    ``accuracy_level`` attribute: unset/0 keeps the highest-precision fp32
+    dequant + fp32 GEMM path, while ``4`` dynamically quantizes activations to
+    int8 and uses int8 dot-products (SDOT/NEON on ARM, AVX-VNNI on x86) — the
+    same class of kernel llama.cpp uses, and typically 2-4x faster on CPU with
+    no observable quality loss for Q4 weights. The value is sourced from the
+    active EP's :attr:`EpCapabilities.default_int4_accuracy_level` (4 for CPU /
+    WebGPU). When it is 0 the attribute is omitted so ORT keeps its default.
+    """
+    if bits != 4:
+        return {}
+    level = ep_capabilities().default_int4_accuracy_level
+    return {"accuracy_level": level} if level else {}
+
+
 class QuantizedLinear(nn.Module):
     """Linear layer backed by the MatMulNBits custom op.
 
@@ -126,6 +151,7 @@ def forward(self, op: OpBuilder, x: ir.Value) -> ir.Value:
             N=self._n,
             bits=self._bits,
             block_size=self._block_size,
+            **_accuracy_level_attrs(self._bits),
             _domain=_MICROSOFT_DOMAIN,
         )
         if self.bias is not None:
@@ -309,6 +335,7 @@ def forward(self, op: OpBuilder, x: ir.Value) -> ir.Value:
             N=self._n,
             bits=self._bits,
             block_size=self._block_size,
+            **_accuracy_level_attrs(self._bits),
             _domain=_MICROSOFT_DOMAIN,
         )
 

src/mobius/components/_quantized_linear_test.py

@@ -9,6 +9,8 @@
 
 import pytest
 
+from mobius._build_context import build_context
+from mobius._execution_providers import ep_registry
 from mobius._testing import (
     count_op_type,
     create_test_builder,
@@ -125,7 +127,60 @@ def test_matmulnbits_attributes(self):
                 assert attrs["N"] == OUT_FEATURES
                 assert attrs["bits"] == 4
                 assert attrs["block_size"] == 32
+                # No active build context -> default EP (accuracy_level 0),
+                # so the attribute is omitted and ORT keeps its default path.
+                assert "accuracy_level" not in attrs
                 break
+        else:
+            pytest.fail("MatMulNBits node not found")
+
+    def test_no_accuracy_level_without_context(self):
+        """Default EP omits accuracy_level (ORT default / highest precision)."""
+        ql = QuantizedLinear(IN_FEATURES, OUT_FEATURES, bits=4, block_size=32)
+        b, op, graph = create_test_builder()
+        x = create_test_input(b, "x", [1, 4, IN_FEATURES])
+        result = ql(op, x)
+        b._adapt_outputs([result], "")
+        for node in graph:
+            if node.op_type == "MatMulNBits":
+                attrs = {a.name: a.value for a in node.attributes.values()}
+                assert "accuracy_level" not in attrs
+                break
+        else:
+            pytest.fail("MatMulNBits node not found")
+
+    def test_cpu_ep_emits_accuracy_level_4(self):
+        """CPU EP context stamps accuracy_level=4 (int8 MLAS path)."""
+        with build_context(ep_registry.require("cpu")):
+            ql = QuantizedLinear(IN_FEATURES, OUT_FEATURES, bits=4, block_size=32)
+            b, op, graph = create_test_builder()
+            x = create_test_input(b, "x", [1, 4, IN_FEATURES])
+            result = ql(op, x)
+            b._adapt_outputs([result], "")
+        for node in graph:
+            if node.op_type == "MatMulNBits":
+                attrs = {a.name: a.value for a in node.attributes.values()}
+                assert attrs["accuracy_level"] == 4
+                break
+        else:
+            pytest.fail("MatMulNBits node not found")
+
+    def test_cpu_ep_omits_accuracy_level_for_non_int4(self):
+        """accuracy_level is INT4-specific: 8-bit weights keep ORT's default."""
+        with build_context(ep_registry.require("cpu")):
+            ql = QuantizedLinear(IN_FEATURES, OUT_FEATURES, bits=8, block_size=32)
+            b, op, graph = create_test_builder()
+            x = create_test_input(b, "x", [1, 4, IN_FEATURES])
+            result = ql(op, x)
+            b._adapt_outputs([result], "")
+        for node in graph:
+            if node.op_type == "MatMulNBits":
+                attrs = {a.name: a.value for a in node.attributes.values()}
+                assert attrs["bits"] == 8
+                assert "accuracy_level" not in attrs
+                break
+        else:
+            pytest.fail("MatMulNBits node not found")
 
     def test_3_inputs_without_zero_points(self):
         ql = QuantizedLinear(IN_FEATURES, OUT_FEATURES)