From 9e8aa8ed38028a8567a71d47ef1a8a2d8f280876 Mon Sep 17 00:00:00 2001 From: moraxu Date: Fri, 16 May 2025 07:00:18 +0000 Subject: [PATCH 1/5] Add accuracy CLI TestLlama4MaverickInstruct test Signed-off-by: moraxu --- .../defs/accuracy/references/mmlu.yaml | 3 + .../defs/accuracy/test_cli_flow.py | 62 +++++++++++++++++++ .../defs/accuracy/test_llm_api_pytorch.py | 26 ++++++++ .../test_lists/qa/examples_test_list.txt | 9 +++ .../test_lists/qa/llm_sanity_test.txt | 3 + 5 files changed, 103 insertions(+) diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 86a07220237e..22e40c3819ba 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -66,6 +66,9 @@ meta-llama/Llama-3.3-70B-Instruct: accuracy: 81.02 meta-llama/Llama-4-Maverick-17B-128E-Instruct: - accuracy: 86.40 + # - quant_algo: FP8_PER_CHANNEL_PER_TOKEN + # kv_cache_quant_algo: FP8 + # accuracy: 0.0 meta-llama/Llama-4-Scout-17B-16E-Instruct: - accuracy: 80.00 - quant_algo: NVFP4 diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index a5ab844dfbc1..ff90c4ea1f4e 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -728,6 +728,68 @@ def test_long_context_ppl(self): extra_build_args=["--gather_context_logits"]) +class TestLlama4MaverickInstruct(CliFlowAccuracyTestHarness): + MODEL_NAME = "meta-llama/Llama-4-Maverick-17B-128E-Instruct" + MODEL_PATH = f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct" + EXAMPLE_FOLDER = "models/core/llama" + + @skip_pre_hopper + @pytest.mark.skip_less_device(8) + @parametrize_with_ids("cuda_graph", [False, True]) + @pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4), + (8, 1, 8)], + ids=["tp8", "tp8ep4", "tp8ep8"]) + def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): + extra_summarize_args = [] + if cuda_graph: + extra_summarize_args.append("--cuda_graph_mode") + + self.run(tasks=[MMLU(self.MODEL_NAME)], + tp_size=tp_size, + pp_size=pp_size, + extra_convert_args=[ + f"--moe_tp_size={tp_size // ep_size}", + f"--moe_ep_size={ep_size}", f"--moe_renorm_mode={0}" + ], + extra_build_args=[ + "--gemm_plugin=auto", "--moe_plugin=auto", + f"--max_seq_len={8192}" + ], + extra_summarize_args=extra_summarize_args) + + @skip_pre_hopper + @pytest.mark.skip_less_device(8) + @parametrize_with_ids("cuda_graph", [False, True]) + @pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4), + (8, 1, 8)], + ids=["tp8", "tp8ep4", "tp8ep8"]) + def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size, + mocker): + mocker.patch.object( + self.__class__, "MODEL_PATH", + f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct-FP8" + ) + + extra_summarize_args = [] + if cuda_graph: + extra_summarize_args.append("--cuda_graph_mode") + + self.run(tasks=[MMLU(self.MODEL_NAME)], + quant_algo=QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, + kv_cache_quant_algo=QuantAlgo.FP8, + tp_size=tp_size, + pp_size=pp_size, + extra_convert_args=[ + f"--moe_tp_size={tp_size // ep_size}", + f"--moe_ep_size={ep_size}", f"--moe_renorm_mode={0}" + ], + extra_build_args=[ + "--gemm_plugin=auto", "--moe_plugin=auto", + f"--max_seq_len={8192}" + ], + extra_summarize_args=extra_summarize_args) + + class TestLlama3_1_8B(CliFlowAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-3.1-8B" MODEL_PATH = f"{llm_models_root()}/llama-3.1-model/Meta-Llama-3.1-8B" diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 204094787043..80a3b9127eac 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -408,6 +408,32 @@ def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + @skip_pre_hopper + @pytest.mark.skip_less_device(8) + @parametrize_with_ids("cuda_graph", [False, True]) + @pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4), + (8, 1, 8)], + ids=["tp8", "tp8ep4", "tp8ep8"]) + def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size): + pytorch_config = PyTorchConfig(use_cuda_graph=cuda_graph) + + quant_config = QuantConfig() + quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN + quant_config.kv_cache_quant_algo = QuantAlgo.FP8 + pytorch_config.kv_cache_dtype = "fp8" + + model_path = f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct-FP8" + with LLM(model_path, + tensor_parallel_size=tp_size, + pipeline_parallel_size=pp_size, + moe_expert_parallel_size=ep_size, + pytorch_backend_config=pytorch_config, + quant_config=quant_config) as llm: + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + class TestLlama4ScoutInstruct(LlmapiAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-4-Scout-17B-16E-Instruct" diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index 3a2c8c2e9820..b5555e1158f0 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -452,6 +452,15 @@ accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep4-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep8-cuda_graph=True] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep4-cuda_graph=True] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep8-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] diff --git a/tests/integration/test_lists/qa/llm_sanity_test.txt b/tests/integration/test_lists/qa/llm_sanity_test.txt index 4c01e492e1b9..2cfda934e59a 100644 --- a/tests/integration/test_lists/qa/llm_sanity_test.txt +++ b/tests/integration/test_lists/qa/llm_sanity_test.txt @@ -35,6 +35,9 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4 accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] +accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] From f07189f814af8cecaaca5217d22e4608446180a6 Mon Sep 17 00:00:00 2001 From: moraxu Date: Sun, 18 May 2025 05:41:20 +0000 Subject: [PATCH 2/5] Update llama config Signed-off-by: moraxu --- tensorrt_llm/llmapi/llm_utils.py | 13 +++++++++++++ tensorrt_llm/models/llama/config.py | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index a62568a54e86..af4600402751 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -426,6 +426,19 @@ def _update_from_hf_quant_config(self) -> bool: "weight_block_size"): quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES quant_config.exclude_modules = ["*eh_proj"] + elif hf_quant_config.get( + "quant_method") == "compressed-tensors": + config_groups = hf_quant_config.get("config_groups", {}) + if "group_0" in config_groups: + group = config_groups["group_0"] + weights_config = group.get("weights", {}) + activations_config = group.get("input_activations", {}) + + if (weights_config.get("strategy") == "channel" and + activations_config.get("strategy") == "token" + and weights_config.get("type") == "float" + and weights_config.get("num_bits") == 8): + quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN else: raise NotImplementedError( f"Unsupported quantization_config: {hf_quant_config}.") diff --git a/tensorrt_llm/models/llama/config.py b/tensorrt_llm/models/llama/config.py index 7e0369a4ba0e..9840dc0be77c 100644 --- a/tensorrt_llm/models/llama/config.py +++ b/tensorrt_llm/models/llama/config.py @@ -142,6 +142,11 @@ def from_hugging_face( hf_config = Mistral3Config.from_pretrained( hf_config_dir).text_config hf_config.architectures = ["MistralForCausalLM"] + if hf_config.model_type == 'llama4': + from transformers import Llama4Config + hf_config = Llama4Config.from_pretrained( + hf_config_dir).text_config + hf_config.architectures = ["Llama4ForConditionalGeneration"] num_key_value_heads = getattr(hf_config, "num_key_value_heads", hf_config.num_attention_heads) From cf4f93fe4320b8a8af4aeaf85d4a43ff9c5ff5ce Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Wed, 21 May 2025 06:31:55 +0000 Subject: [PATCH 3/5] Address review comments Signed-off-by: Michal Guzek --- .../defs/accuracy/test_cli_flow.py | 62 ------------------- .../defs/accuracy/test_llm_api_pytorch.py | 8 +-- .../test_lists/qa/examples_test_list.txt | 9 --- 3 files changed, 2 insertions(+), 77 deletions(-) diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index ff90c4ea1f4e..a5ab844dfbc1 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -728,68 +728,6 @@ def test_long_context_ppl(self): extra_build_args=["--gather_context_logits"]) -class TestLlama4MaverickInstruct(CliFlowAccuracyTestHarness): - MODEL_NAME = "meta-llama/Llama-4-Maverick-17B-128E-Instruct" - MODEL_PATH = f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct" - EXAMPLE_FOLDER = "models/core/llama" - - @skip_pre_hopper - @pytest.mark.skip_less_device(8) - @parametrize_with_ids("cuda_graph", [False, True]) - @pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4), - (8, 1, 8)], - ids=["tp8", "tp8ep4", "tp8ep8"]) - def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): - extra_summarize_args = [] - if cuda_graph: - extra_summarize_args.append("--cuda_graph_mode") - - self.run(tasks=[MMLU(self.MODEL_NAME)], - tp_size=tp_size, - pp_size=pp_size, - extra_convert_args=[ - f"--moe_tp_size={tp_size // ep_size}", - f"--moe_ep_size={ep_size}", f"--moe_renorm_mode={0}" - ], - extra_build_args=[ - "--gemm_plugin=auto", "--moe_plugin=auto", - f"--max_seq_len={8192}" - ], - extra_summarize_args=extra_summarize_args) - - @skip_pre_hopper - @pytest.mark.skip_less_device(8) - @parametrize_with_ids("cuda_graph", [False, True]) - @pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4), - (8, 1, 8)], - ids=["tp8", "tp8ep4", "tp8ep8"]) - def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size, - mocker): - mocker.patch.object( - self.__class__, "MODEL_PATH", - f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct-FP8" - ) - - extra_summarize_args = [] - if cuda_graph: - extra_summarize_args.append("--cuda_graph_mode") - - self.run(tasks=[MMLU(self.MODEL_NAME)], - quant_algo=QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, - kv_cache_quant_algo=QuantAlgo.FP8, - tp_size=tp_size, - pp_size=pp_size, - extra_convert_args=[ - f"--moe_tp_size={tp_size // ep_size}", - f"--moe_ep_size={ep_size}", f"--moe_renorm_mode={0}" - ], - extra_build_args=[ - "--gemm_plugin=auto", "--moe_plugin=auto", - f"--max_seq_len={8192}" - ], - extra_summarize_args=extra_summarize_args) - - class TestLlama3_1_8B(CliFlowAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-3.1-8B" MODEL_PATH = f"{llm_models_root()}/llama-3.1-model/Meta-Llama-3.1-8B" diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 80a3b9127eac..3f57dc358adc 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -408,6 +408,7 @@ def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + @pytest.mark.skip(reason="OOM: https://nvbugspro.nvidia.com/bug/5295255") @skip_pre_hopper @pytest.mark.skip_less_device(8) @parametrize_with_ids("cuda_graph", [False, True]) @@ -416,10 +417,6 @@ def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): ids=["tp8", "tp8ep4", "tp8ep8"]) def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size): pytorch_config = PyTorchConfig(use_cuda_graph=cuda_graph) - - quant_config = QuantConfig() - quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN - quant_config.kv_cache_quant_algo = QuantAlgo.FP8 pytorch_config.kv_cache_dtype = "fp8" model_path = f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct-FP8" @@ -427,8 +424,7 @@ def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size): tensor_parallel_size=tp_size, pipeline_parallel_size=pp_size, moe_expert_parallel_size=ep_size, - pytorch_backend_config=pytorch_config, - quant_config=quant_config) as llm: + pytorch_backend_config=pytorch_config) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm) task = GSM8K(self.MODEL_NAME) diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index b5555e1158f0..3a2c8c2e9820 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -452,15 +452,6 @@ accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep4-cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep8-cuda_graph=True] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8-cuda_graph=False] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8-cuda_graph=False] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep4-cuda_graph=True] -accuracy/test_cli_flow.py::TestLlama4MaverickInstruct::test_fp8_prequantized[tp8ep8-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-cuda_graph=False] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep4-cuda_graph=True] accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8ep8-cuda_graph=True] From 9121cd9968b014c798e5d6a0868887d87590bc3e Mon Sep 17 00:00:00 2001 From: moraxu Date: Wed, 21 May 2025 06:36:15 +0000 Subject: [PATCH 4/5] Remove changes for trt-related configs Signed-off-by: moraxu --- tensorrt_llm/llmapi/llm_utils.py | 13 ------------- tensorrt_llm/models/llama/config.py | 5 ----- 2 files changed, 18 deletions(-) diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index af4600402751..a62568a54e86 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -426,19 +426,6 @@ def _update_from_hf_quant_config(self) -> bool: "weight_block_size"): quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES quant_config.exclude_modules = ["*eh_proj"] - elif hf_quant_config.get( - "quant_method") == "compressed-tensors": - config_groups = hf_quant_config.get("config_groups", {}) - if "group_0" in config_groups: - group = config_groups["group_0"] - weights_config = group.get("weights", {}) - activations_config = group.get("input_activations", {}) - - if (weights_config.get("strategy") == "channel" and - activations_config.get("strategy") == "token" - and weights_config.get("type") == "float" - and weights_config.get("num_bits") == 8): - quant_config.quant_algo = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN else: raise NotImplementedError( f"Unsupported quantization_config: {hf_quant_config}.") diff --git a/tensorrt_llm/models/llama/config.py b/tensorrt_llm/models/llama/config.py index 9840dc0be77c..7e0369a4ba0e 100644 --- a/tensorrt_llm/models/llama/config.py +++ b/tensorrt_llm/models/llama/config.py @@ -142,11 +142,6 @@ def from_hugging_face( hf_config = Mistral3Config.from_pretrained( hf_config_dir).text_config hf_config.architectures = ["MistralForCausalLM"] - if hf_config.model_type == 'llama4': - from transformers import Llama4Config - hf_config = Llama4Config.from_pretrained( - hf_config_dir).text_config - hf_config.architectures = ["Llama4ForConditionalGeneration"] num_key_value_heads = getattr(hf_config, "num_key_value_heads", hf_config.num_attention_heads) From 8c67f1e2ec2e34ddfe67c39278fbea9893f58ba4 Mon Sep 17 00:00:00 2001 From: moraxu Date: Thu, 24 Jul 2025 17:11:12 -0700 Subject: [PATCH 5/5] Update the test Signed-off-by: moraxu --- tests/integration/defs/accuracy/references/mmlu.yaml | 6 +++--- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 22e40c3819ba..21bb5798ccbc 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -66,9 +66,9 @@ meta-llama/Llama-3.3-70B-Instruct: accuracy: 81.02 meta-llama/Llama-4-Maverick-17B-128E-Instruct: - accuracy: 86.40 - # - quant_algo: FP8_PER_CHANNEL_PER_TOKEN - # kv_cache_quant_algo: FP8 - # accuracy: 0.0 + - quant_algo: FP8 + kv_cache_quant_algo: FP8 + accuracy: 0.0 meta-llama/Llama-4-Scout-17B-16E-Instruct: - accuracy: 80.00 - quant_algo: NVFP4 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 3f57dc358adc..23665df35af9 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -408,7 +408,6 @@ def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): task = GSM8K(self.MODEL_NAME) task.evaluate(llm) - @pytest.mark.skip(reason="OOM: https://nvbugspro.nvidia.com/bug/5295255") @skip_pre_hopper @pytest.mark.skip_less_device(8) @parametrize_with_ids("cuda_graph", [False, True]) @@ -416,15 +415,13 @@ def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size): (8, 1, 8)], ids=["tp8", "tp8ep4", "tp8ep8"]) def test_fp8_prequantized(self, cuda_graph, tp_size, pp_size, ep_size): - pytorch_config = PyTorchConfig(use_cuda_graph=cuda_graph) - pytorch_config.kv_cache_dtype = "fp8" - - model_path = f"{llm_models_root()}/llama4-models/Llama-4-Maverick-17B-128E-Instruct-FP8" + model_path = f"{llm_models_root()}/llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8" with LLM(model_path, tensor_parallel_size=tp_size, pipeline_parallel_size=pp_size, moe_expert_parallel_size=ep_size, - pytorch_backend_config=pytorch_config) as llm: + cuda_graph_config=CudaGraphConfig() + if cuda_graph else None) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm) task = GSM8K(self.MODEL_NAME)