Skip to content

Commit 4082f1c

Browse files
committed
Emit consolidated Gemma4Audio op for gemma-4 audio configs
Follow the onnxruntime-extensions consolidation: both gemma4 audio configs now use the single Gemma4Audio op with an explicit type attribute instead of two separate op types. * gemma4 / gemma4_text -> Gemma4Audio type="log_mel" (128-dim USM log-mel) * gemma4_unified* -> Gemma4Audio type="raw_frames" (raw 640-sample frames) Updates the auto_export audio-config tests accordingly. 115 tests pass. Signed-off-by: Justin Chu <justinchuby@users.noreply.github.com>
1 parent 52dfb2b commit 4082f1c

2 files changed

Lines changed: 17 additions & 11 deletions

File tree

src/mobius/integrations/ort_genai/auto_export.py

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -667,8 +667,8 @@ def _write_audio_processor_config(
667667
# Encoder-free unified model: each audio soft token is a raw chunk of the
668668
# 16 kHz waveform (audio_samples_per_token = audio_embed_dim, 640), not a
669669
# 128-dim log-mel frame. Reproduced natively by the ort-extensions
670-
# ``Gemma4UnifiedAudioFrames`` op (pad to a whole number of frames,
671-
# reshape to (num_tokens, 640)).
670+
# ``Gemma4Audio`` op with ``type="raw_frames"`` (pad to a whole number of
671+
# frames, reshape to (num_tokens, 640)).
672672
samples_per_token = getattr(audio, "hidden_size", None) or 640
673673
processor = {
674674
"feature_extraction": {
@@ -681,9 +681,10 @@ def _write_audio_processor_config(
681681
},
682682
{
683683
"operation": {
684-
"name": "gemma4_unified_audio_frames",
685-
"type": "Gemma4UnifiedAudioFrames",
684+
"name": "gemma4_audio",
685+
"type": "Gemma4Audio",
686686
"attrs": {
687+
"type": "raw_frames",
687688
"audio_samples_per_token": samples_per_token,
688689
"sampling_rate": 16000,
689690
"padding_value": 0.0,
@@ -695,7 +696,8 @@ def _write_audio_processor_config(
695696
}
696697
proc_filename = "audio_feature_extraction.json"
697698
elif model_type in _GEMMA4_MODEL_TYPES:
698-
# Gemma4 USM-style 128-dim log-mel spectrogram.
699+
# Gemma4 USM-style 128-dim log-mel spectrogram via the ort-extensions
700+
# ``Gemma4Audio`` op with ``type="log_mel"``.
699701
# OrtxCreateSpeechFeatureExtractor requires the feature_extraction.sequence format.
700702
processor = {
701703
"feature_extraction": {
@@ -708,9 +710,10 @@ def _write_audio_processor_config(
708710
},
709711
{
710712
"operation": {
711-
"name": "gemma4_log_mel",
712-
"type": "Gemma4LogMel",
713+
"name": "gemma4_audio",
714+
"type": "Gemma4Audio",
713715
"attrs": {
716+
"type": "log_mel",
714717
"feature_size": 128,
715718
"sampling_rate": 16000,
716719
"frame_length_ms": 20.0,

src/mobius/integrations/ort_genai/auto_export_test.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,8 @@ def test_audio_gemma4_unified_writes_raw_frames(self, tmp_path):
357357
seq = data["feature_extraction"]["sequence"]
358358
assert seq[0]["operation"]["type"] == "AudioDecoder"
359359
op = seq[1]["operation"]
360-
assert op["type"] == "Gemma4UnifiedAudioFrames"
360+
assert op["type"] == "Gemma4Audio"
361+
assert op["attrs"]["type"] == "raw_frames"
361362
assert op["attrs"]["audio_samples_per_token"] == 640
362363

363364
def test_audio_gemma4_writes_feature_extraction_json(self, tmp_path):
@@ -376,8 +377,9 @@ def test_audio_gemma4_writes_feature_extraction_json(self, tmp_path):
376377
seq = data["feature_extraction"]["sequence"]
377378
assert len(seq) == 2
378379
assert seq[0]["operation"]["type"] == "AudioDecoder"
379-
assert seq[1]["operation"]["type"] == "Gemma4LogMel"
380+
assert seq[1]["operation"]["type"] == "Gemma4Audio"
380381
attrs = seq[1]["operation"]["attrs"]
382+
assert attrs["type"] == "log_mel"
381383
assert attrs["feature_size"] == 128
382384
assert attrs["sampling_rate"] == 16000
383385
assert attrs["frame_length_ms"] == 20.0 # noqa: RUF069
@@ -747,9 +749,10 @@ class FakeConfig:
747749
op0 = seq[0]["operation"]
748750
assert op0["type"] == "AudioDecoder"
749751

750-
# Second op: Gemma4LogMel with expected attrs
752+
# Second op: Gemma4Audio (type=log_mel) with expected attrs
751753
op1 = seq[1]["operation"]
752-
assert op1["type"] == "Gemma4LogMel"
754+
assert op1["type"] == "Gemma4Audio"
755+
assert op1["attrs"]["type"] == "log_mel"
753756
assert op1["attrs"]["feature_size"] == 128
754757
assert op1["attrs"]["sampling_rate"] == 16000
755758
assert op1["attrs"]["mel_floor"] == 0.001 # noqa: RUF069

0 commit comments

Comments
 (0)