chore: add EXAONE4 accuracy test

tensorrt_llm/_torch/models/modeling_exaone4.py

@@ -3,7 +3,6 @@
 import torch
 from torch import nn
 
-from tensorrt_llm._torch.distributed import AllReduceParams
 from tensorrt_llm.functional import PositionEmbeddingType
 
 from ..attention_backend import AttentionMetadata
@@ -55,7 +54,6 @@ class Exaone4Attention(Attention):
 
     def __init__(self,
                  model_config: ModelConfig[Exaone4Config],
-                 is_sliding: bool,
                  layer_idx: Optional[int] = None,
                  aux_stream: Optional[torch.cuda.Stream] = None,
                  fuse_qk_norm_rope: bool = False):
@@ -64,9 +62,10 @@ def __init__(self,
         self.attention_window_size = None
 
         # NOTE: In EXAONE4, only sliding layers apply rope.
-        self.is_sliding = is_sliding
+        self.sliding_window = config.sliding_window
+        self.is_sliding = check_is_sliding(config, layer_idx)
         pos_embd_params = None
-        if self.is_sliding:
+        if self.sliding_window is None or self.is_sliding:
             self.attention_window_size = config.sliding_window
 
             pos_embd_params = PositionalEmbeddingParams(
@@ -140,7 +139,7 @@ def apply_rope(self, q: torch.Tensor, k: Optional[torch.Tensor],
 
         q, k, v = self.split_qkv(q, k, v)
         q, k = self.apply_qk_norm(q, k)
-        if self.is_sliding:
+        if self.sliding_window is None or self.is_sliding:
             return super().apply_rope(q, k, v, position_ids)
         else:
             return q, k, v
@@ -152,7 +151,6 @@ def forward(
         attn_metadata: AttentionMetadata,
         attention_mask: PredefinedAttentionMask = PredefinedAttentionMask.
         CAUSAL,
-        all_reduce_params: Optional[AllReduceParams] = None,
         lora_params: Optional[dict] = None,
         **kwargs,
     ) -> torch.Tensor:
@@ -165,7 +163,6 @@ def forward(
             hidden_states=hidden_states,
             attn_metadata=attn_metadata,
             attention_mask=attention_mask,
-            all_reduce_params=all_reduce_params,
             lora_params=lora_params,
             attention_window_size=self.attention_window_size,
             **kwargs,
@@ -185,11 +182,9 @@ def __init__(
         self.layer_idx = layer_idx
         self.is_quanted = model_config.quant_config and model_config.quant_config.quant_mode.has_any_quant(
         )
-        is_sliding = check_is_sliding(config, layer_idx)
 
         self.self_attn = Exaone4Attention(
             model_config,
-            is_sliding=is_sliding,
             layer_idx=layer_idx,
             aux_stream=aux_stream,
         )
@@ -228,20 +223,14 @@ def forward(
             position_ids=position_ids,
             hidden_states=hidden_states,
             attn_metadata=attn_metadata,
-            all_reduce_params=AllReduceParams(
-                enable_allreduce=not (self.mapping.tp_size == 1)),
             **kwargs,
         )
 
         hidden_states = self.post_attention_layernorm(hidden_states)
         hidden_states = residual + hidden_states
         residual = hidden_states
 
-        hidden_states = self.mlp(
-            hidden_states,
-            final_all_reduce_params=AllReduceParams(
-                enable_allreduce=not (self.mapping.tp_size == 1)),
-        )
+        hidden_states = self.mlp(hidden_states)
         hidden_states = self.post_feedforward_layernorm(hidden_states)
 
         hidden_states = hidden_states + residual

tests/integration/defs/accuracy/references/gsm8k.yaml

@@ -145,3 +145,5 @@ microsoft/Phi-4-multimodal-instruct-long-rope:
   - accuracy: 75.85
 microsoft/Phi-4-mini-instruct:
   - accuracy: 82.30
+LGAI-EXAONE/EXAONE-4.0-32B:
+  - accuracy: 88.36

tests/integration/defs/accuracy/references/mmlu.yaml

@@ -232,3 +232,5 @@ microsoft/Phi-4-multimodal-instruct:
   - accuracy: 69.69
 microsoft/Phi-4-multimodal-instruct-long-rope:
   - accuracy: 65.98
+LGAI-EXAONE/EXAONE-4.0-32B:
+  - accuracy: 78.52

tests/integration/defs/accuracy/test_llm_api_pytorch.py

@@ -2103,3 +2103,19 @@ def test_auto_dtype_long_rope(self):
             task.evaluate(llm)
             task = GSM8K(model_name)
             task.evaluate(llm)
+
+
+class TestEXAONE4(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "LGAI-EXAONE/EXAONE-4.0-32B"
+    kv_cache_config = KvCacheConfig(
+        enable_block_reuse=False,
+        enable_partial_reuse=False,
+        max_attention_window=[4096, 4096, 4096, 131072])
+
+    def test_auto_dtype(self):
+        model_path = f"{llm_models_root()}/EXAONE-4.0-32B"
+        with LLM(model_path, kv_cache_config=self.kv_cache_config) as llm:
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm)
+            task = GSM8K(self.MODEL_NAME)
+            task.evaluate(llm)

tests/integration/test_lists/qa/examples_test_list.txt

@@ -497,6 +497,7 @@ accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8
 accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestPhi4MM::test_auto_dtype_long_rope
 accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype
+accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype
 
 test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding-]
 test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding-]

tests/integration/test_lists/test-db/l0_a30.yml

@@ -18,6 +18,7 @@ l0_a30:
   - unittest/_torch/modeling -k "modeling_phi3"
   - unittest/_torch/modeling -k "modeling_qwen"
   - unittest/_torch/modeling -k "modeling_qwen_moe"
+  - unittest/_torch/modeling -k "modeling_exaone4"
   - unittest/_torch/auto_deploy/unit/singlegpu
   - unittest/_torch/test_beam_search.py
 - condition:

tests/integration/test_lists/test-db/l0_h100.yml

@@ -43,6 +43,7 @@ l0_h100:
   - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3
   - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]
   - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]
+
   - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]
   - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]
   - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8]