Fix ORT GenAI export metadata (#377)

## Summary
- copy tokenizer files locally when hf_model_id is a local directory
- emit qwen3_vl model type instead of qwen2_5_vl for Qwen3-VL exports
- include Qwen3-VL vision/video metadata in genai_config.json

## Validation
- PYTHONPATH=$PWD/src conda run -n olive python -m pytest
src/mobius/integrations/ort_genai/auto_export_test.py -q -k
'local_hf_model_id_uses_local_tokenizer_copy or
qwen3_vl_writes_qwen3_vl_model_type_and_vision_fields or
writes_processor_config_for_vision_language_model'

---------

Signed-off-by: Xiaoyu <xiaoyuzhang@microsoft.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: 3 people

src/mobius/integrations/ort_genai/auto_export.py

@@ -87,10 +87,10 @@
     # ORT GenAI (see onnxruntime-genai/src/models/model_type.h LLM list).
     "hunyuan_v1_dense": "decoder",
     "deepseek_v4": "decoder",
-    # Qwen VL models all use the same GenAI pipeline as qwen2_5_vl
+    # Qwen VL model families have separate ORT GenAI model types.
     "qwen2_vl": "qwen2_5_vl",
-    "qwen3_vl": "qwen2_5_vl",
-    "qwen3_vl_text": "qwen2_5_vl",
+    "qwen3_vl": "qwen3_vl",
+    "qwen3_vl_text": "qwen3_vl",
     "qwen3_5": "qwen2_5_vl",
     "qwen3_5_vl": "qwen2_5_vl",
 }
@@ -880,6 +880,16 @@ def _write_genai_config(
                 if sms is not None:
                     vision_kwargs["spatial_merge_size"] = sms
                 vision_kwargs["config_filename"] = "processor_config.json"
+                if model_type in {"qwen3_vl", "qwen3_vl_text"}:
+                    patch_size = getattr(vision_cfg, "patch_size", None)
+                    window_size = getattr(vision_cfg, "window_size", None)
+                    if patch_size is not None:
+                        vision_kwargs["patch_size"] = patch_size
+                    if window_size is not None:
+                        vision_kwargs["window_size"] = window_size
+                    vision_kwargs["tokens_per_second"] = float(
+                        getattr(config, "tokens_per_second", 2.0)
+                    )
 
             if vision_input_mapping is not None:
                 vision_kwargs["input_names"] = vision_input_mapping
@@ -889,6 +899,12 @@ def _write_genai_config(
             embedding_output_mapping = _introspect_outputs(pkg, "embedding")
             if embedding_output_mapping is not None:
                 vision_kwargs["embedding_output_names"] = embedding_output_mapping
+            vision_start_token_id = getattr(config, "vision_start_token_id", None)
+            video_token_id = getattr(config, "video_token_id", None)
+            if vision_start_token_id is not None:
+                vision_kwargs["vision_start_token_id"] = vision_start_token_id
+            if video_token_id is not None:
+                vision_kwargs["video_token_id"] = video_token_id
 
             generator.with_vision(image_token_id=image_token_id, **vision_kwargs)
 
@@ -946,8 +962,8 @@ def write_ort_genai_config(
         pkg: Already-built :class:`~mobius._model_package.ModelPackage` with
             weights applied and ``config`` set.
         directory: Output directory (created if needed).
-        hf_model_id: HuggingFace model ID. When provided, used to fetch token
-            IDs (``bos``/``eos``/``pad``) and download tokenizer files.
+        hf_model_id: HuggingFace model ID or local model directory. When provided,
+            used to fetch token IDs (``bos``/``eos``/``pad``) and copy tokenizer files.
             When ``None``, token IDs are read from ``pkg.config`` fields
             (``bos_token_id``, ``eos_token_id``, ``pad_token_id``) populated
             by :meth:`~mobius._configs.ArchitectureConfig.from_transformers`,
@@ -1112,11 +1128,15 @@ def write_ort_genai_config(
             f.write("\n")
         result["mtp_config"] = mtp_path
 
-    # Copy tokenizer files — HF Hub takes precedence; local dir is the fallback
-    # for --config mode where no HF model ID is available.
+    # Copy tokenizer files. A local hf_model_id is a local model directory, not a
+    # Hub repo id; copy directly instead of calling hf_hub_download.
     if hf_model_id is not None:
-        logger.info("Copying tokenizer files from %s", hf_model_id)
-        tokenizer_files = _copy_tokenizer_files(hf_model_id, directory)
+        if os.path.isdir(hf_model_id):
+            logger.info("Copying tokenizer files from local model directory %s", hf_model_id)
+            tokenizer_files = _copy_tokenizer_files_from_local(hf_model_id, directory)
+        else:
+            logger.info("Copying tokenizer files from %s", hf_model_id)
+            tokenizer_files = _copy_tokenizer_files(hf_model_id, directory)
         for tf in tokenizer_files:
             result[tf] = os.path.join(directory, tf)
     elif local_config_dir is not None:

src/mobius/integrations/ort_genai/auto_export_test.py

@@ -602,6 +602,36 @@ def test_hf_model_id_takes_precedence_over_local_dir(self, tmp_path):
         mock_hf.assert_called_once()
         mock_local.assert_not_called()
 
+    def test_local_hf_model_id_uses_local_tokenizer_copy(self, tmp_path):
+        """A local hf_model_id should copy tokenizer files locally, not call the Hub."""
+        src = tmp_path / "local_model"
+        src.mkdir()
+        (src / "config.json").write_text(
+            '{"model_type": "llama", "bos_token_id": 1, "eos_token_id": 2}'
+        )
+        (src / "tokenizer.json").write_text('{"local": true}')
+
+        out = tmp_path / "output"
+        out.mkdir()
+        pkg = self._make_pkg()
+
+        with (
+            mock.patch(
+                "mobius.integrations.ort_genai.auto_export._copy_tokenizer_files",
+                return_value=[],
+            ) as mock_hub_copy,
+            mock.patch(
+                "mobius.integrations.ort_genai.auto_export._copy_tokenizer_files_from_local",
+                wraps=_copy_tokenizer_files_from_local,
+            ) as mock_local_copy,
+        ):
+            result = write_ort_genai_config(pkg, str(out), hf_model_id=str(src))
+
+        mock_hub_copy.assert_not_called()
+        mock_local_copy.assert_called_once_with(str(src), str(out))
+        assert "tokenizer.json" in result
+        assert (out / "tokenizer.json").read_text() == '{"local": true}'
+
 
 class TestExportForOrtGenai:
     """Unit tests for write_ort_genai_config()."""
@@ -656,7 +686,7 @@ class FakeConfig:
             },
             config=FakeConfig(),
         )
-        result = write_ort_genai_config(pkg, str(tmp_path))
+        result = write_ort_genai_config(pkg, str(tmp_path), ep="cuda")
 
         assert "processor_config" in result
         assert os.path.isfile(result["processor_config"])
@@ -670,6 +700,57 @@ class FakeConfig:
         resize = transforms[2]["operation"]["attrs"]
         assert resize["patch_size"] == 14
 
+    def test_qwen3_vl_writes_qwen3_vl_model_type_and_vision_fields(self, tmp_path):
+        import dataclasses
+
+        from mobius._model_package import ModelPackage
+        from mobius.integrations.ort_genai.auto_export import write_ort_genai_config
+
+        @dataclasses.dataclass
+        class FakeVision:
+            image_size: int = 448
+            patch_size: int = 16
+            spatial_merge_size: int = 2
+            window_size: int = 64
+            model_type: str | None = None
+
+        @dataclasses.dataclass
+        class FakeConfig:
+            model_type: str = "qwen3_vl"
+            vocab_size: int = 151936
+            hidden_size: int = 2048
+            num_hidden_layers: int = 1
+            num_attention_heads: int = 16
+            num_key_value_heads: int = 8
+            head_dim: int = 128
+            image_token_id: int = 151655
+            vision_start_token_id: int = 151652
+            video_token_id: int = 151656
+            tokens_per_second: float = 2.0
+            temporal_patch_size: int = 2
+            vision: FakeVision = dataclasses.field(default_factory=FakeVision)
+
+        pkg = ModelPackage(
+            {
+                "decoder": mock.MagicMock(),
+                "vision_encoder": mock.MagicMock(),
+                "embedding": mock.MagicMock(),
+            },
+            config=FakeConfig(),
+        )
+
+        result = write_ort_genai_config(pkg, str(tmp_path), ep="cuda")
+
+        with open(result["genai_config"]) as f:
+            data = json.load(f)
+        model = data["model"]
+        assert model["type"] == "qwen3_vl"
+        assert model["vision_start_token_id"] == 151652
+        assert model["video_token_id"] == 151656
+        assert model["vision"]["tokens_per_second"] == pytest.approx(2.0)
+        assert model["vision"]["patch_size"] == 16
+        assert model["vision"]["window_size"] == 64
+
     def test_processor_config_not_written_without_vision(self, tmp_path):
         """image_processor.json is NOT written when pkg.config has no vision attr."""
         from mobius.integrations.ort_genai.auto_export import write_ort_genai_config

src/mobius/integrations/ort_genai/genai_config.py

@@ -269,6 +269,9 @@ def with_vision(
         embedding_output_names: dict[str, str] | None = None,
         vision_start_token_id: int | None = None,
         video_token_id: int | None = None,
+        tokens_per_second: float | None = None,
+        patch_size: int | None = None,
+        window_size: int | None = None,
     ) -> GenaiConfigGenerator:
         """Add VLM vision + embedding sections.
 
@@ -293,6 +296,9 @@ def with_vision(
                 mapping. Defaults to inputs_embeds.
             vision_start_token_id: Token ID for ``<|vision_start|>``.
             video_token_id: Token ID for video placeholders.
+            tokens_per_second: Video/image timestamp rate for Qwen3-VL.
+            patch_size: Vision patch size.
+            window_size: Vision window size.
 
         Returns self for chaining.
         """
@@ -320,6 +326,12 @@ def with_vision(
         }
         if spatial_merge_size is not None:
             self._vision["spatial_merge_size"] = spatial_merge_size
+        if tokens_per_second is not None:
+            self._vision["tokens_per_second"] = tokens_per_second
+        if patch_size is not None:
+            self._vision["patch_size"] = patch_size
+        if window_size is not None:
+            self._vision["window_size"] = window_size
 
         self._embedding = {
             "filename": embedding_filename,

tests/integration_test.py

@@ -3782,10 +3782,10 @@ def test_qwen35_deltanet_single_layer_parity():
             hidden_states=torch.from_numpy(hidden_np).float(),
             cache_params=cache,
         ).numpy()
-    hf_rec = cache.layers[0].recurrent_states
-    if isinstance(hf_rec, dict):
-        hf_rec = hf_rec[0]
-    hf_rec = hf_rec.numpy()
+    # transformers >=5.14 changed recurrent_states from a tensor to a dict
+    # keyed by layer index; extract the tensor for either version.
+    _rec_states = cache.layers[0].recurrent_states
+    hf_rec = (_rec_states[0] if isinstance(_rec_states, dict) else _rec_states).numpy()
 
     # ONNX forward
     sess = _make_session(onnx_model)