From aa9348f6521f69e60c0fe78dc4252baba893b8a3 Mon Sep 17 00:00:00 2001 From: moraxu Date: Thu, 15 May 2025 23:32:36 +0000 Subject: [PATCH 1/6] Add CLI TestNemotronSuper acc tests Signed-off-by: moraxu --- .../defs/accuracy/accuracy_core.py | 3 +++ .../defs/accuracy/references/mmlu.yaml | 2 ++ .../defs/accuracy/test_cli_flow.py | 22 +++++++++++++++++++ .../defs/accuracy/test_llm_api_pytorch.py | 15 +++++++++++++ .../test_lists/qa/examples_test_list.txt | 3 +++ .../test_lists/qa/llm_sanity_test.txt | 1 + 6 files changed, 46 insertions(+) diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index d9c9980d5e61..1ff754f97342 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -434,6 +434,9 @@ def convert(self): f"--dtype={self.dtype}", ] + if "nemotron_nas" in self.EXAMPLE_FOLDER: + convert_cmd.append("--trust_remote_code") + if self.MODEL_FORMAT == "NEMO": convert_cmd.append(f"--nemo_ckpt_path={self.MODEL_PATH}") else: diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 51c1cdd69ea1..d9fa3bc09571 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -121,6 +121,8 @@ Qwen3/Qwen3-30B-A3B: accuracy: 80.65 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 79.43 + # - quant_algo: FP8 + # accuracy: 79.43 nvidia/Llama-3.1-Nemotron-Nano-8B-v1: - accuracy: 57.97 nvidia/Nemotron-H-8B-Base-8K: diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index e9fdac5dcbec..ab8532c3f49c 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -200,6 +200,28 @@ def test_fp8_prequantized(self, mocker): self.run(quant_algo=QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) +# TODO: Remove the CLI tests once NIMs use PyTorch backend +class TestNemotronSuper(CliFlowAccuracyTestHarness): + MODEL_NAME = "nvidia/Llama-3_3-Nemotron-Super-49B-v1" + MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1" + EXAMPLE_FOLDER = "models/core/nemotron_nas" + + @pytest.mark.skip_less_device(2) + def test_auto_dtype_tp2(self): + self.run(tasks=[MMLU(self.MODEL_NAME)], tp_size=2, dtype='auto') + + @pytest.mark.skip_less_device(2) + @pytest.mark.skip_device_not_contain(["H100", "B200"]) + def test_fp8_prequantized_tp2(self, mocker): + mocker.patch.object( + self.__class__, "MODEL_PATH", + f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8" + ) + self.run(tasks=[MMLU(self.MODEL_NAME)], + tp_size=2, + quant_algo=QuantAlgo.FP8) + + class TestPhi2(CliFlowAccuracyTestHarness): MODEL_NAME = "microsoft/phi-2" MODEL_PATH = f"{llm_models_root()}/phi-2" diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index ad8b59eec5c8..abf5d77116ff 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -906,6 +906,21 @@ def test_auto_dtype_tp2(self): task.evaluate(llm, extra_evaluator_kwargs=dict(apply_chat_template=True)) + @pytest.mark.skip_less_device(2) + @pytest.mark.skip_device_not_contain(["H100", "B200"]) + def test_fp8_prequantized_tp2(self): + model_path = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8" + with LLM(model_path, tensor_parallel_size=2) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + # TODO: Add GSM8K and GPQADiamond once the CLI tests are removed + # task = GSM8K(self.MODEL_NAME) + # task.evaluate(llm) + # task = GPQADiamond(self.MODEL_NAME) + # task.evaluate(llm, + # extra_evaluator_kwargs=dict(apply_chat_template=True)) + class TestNemotronNano(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Llama-3.1-Nemotron-Nano-8B-v1" diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index 6dd0b6c834f2..d401068daa29 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -446,6 +446,9 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[fp8kv=False-att accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_fp8_prequantized_tp2 +accuracy/test_cli_flow.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestNemotronSuper::test_fp8_prequantized_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype diff --git a/tests/integration/test_lists/qa/llm_sanity_test.txt b/tests/integration/test_lists/qa/llm_sanity_test.txt index d174a0a1cfc6..d5cc10f8a8e2 100644 --- a/tests/integration/test_lists/qa/llm_sanity_test.txt +++ b/tests/integration/test_lists/qa/llm_sanity_test.txt @@ -137,6 +137,7 @@ accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2 accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestNemotronSuper::test_auto_dtype_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] From 3c417db757122b39feae3d4a5f8d47e16fdce800 Mon Sep 17 00:00:00 2001 From: moraxu Date: Fri, 16 May 2025 00:16:37 +0000 Subject: [PATCH 2/6] Update mmlu.yaml Signed-off-by: moraxu --- tests/integration/defs/accuracy/references/mmlu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index d9fa3bc09571..4380b799764b 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -121,8 +121,8 @@ Qwen3/Qwen3-30B-A3B: accuracy: 80.65 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 79.43 - # - quant_algo: FP8 - # accuracy: 79.43 + - quant_algo: FP8 + accuracy: 79.26 nvidia/Llama-3.1-Nemotron-Nano-8B-v1: - accuracy: 57.97 nvidia/Nemotron-H-8B-Base-8K: From 7e862a818405d6898e780043f8cecdb17b0acffe Mon Sep 17 00:00:00 2001 From: moraxu Date: Fri, 16 May 2025 04:29:43 +0000 Subject: [PATCH 3/6] Update yaml files Signed-off-by: moraxu --- .../defs/accuracy/references/gpqa_diamond.yaml | 2 ++ tests/integration/defs/accuracy/references/gsm8k.yaml | 2 ++ .../integration/defs/accuracy/test_llm_api_pytorch.py | 11 +++++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/integration/defs/accuracy/references/gpqa_diamond.yaml b/tests/integration/defs/accuracy/references/gpqa_diamond.yaml index 3fccc693227e..9e4e988f335d 100644 --- a/tests/integration/defs/accuracy/references/gpqa_diamond.yaml +++ b/tests/integration/defs/accuracy/references/gpqa_diamond.yaml @@ -16,3 +16,5 @@ deepseek-ai/DeepSeek-R1: accuracy: 70.45 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 44.95 + - quant_algo: FP8 + accuracy: 49.49 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index a19abf12e8d3..9748b122bcd1 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -65,5 +65,7 @@ Qwen3/Qwen3-30B-A3B: accuracy: 83.43 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 92.57 + - quant_algo: FP8 + accuracy: 92.42 nvidia/Nemotron-H-8B-Base-8K: - accuracy: 46.20 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index abf5d77116ff..97a560e0c6ad 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -914,12 +914,11 @@ def test_fp8_prequantized_tp2(self): assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 task = MMLU(self.MODEL_NAME) task.evaluate(llm) - # TODO: Add GSM8K and GPQADiamond once the CLI tests are removed - # task = GSM8K(self.MODEL_NAME) - # task.evaluate(llm) - # task = GPQADiamond(self.MODEL_NAME) - # task.evaluate(llm, - # extra_evaluator_kwargs=dict(apply_chat_template=True)) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + task = GPQADiamond(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=dict(apply_chat_template=True)) class TestNemotronNano(LlmapiAccuracyTestHarness): From 6c4990cd6c4220094126cb80b04e0c9ef79d5daa Mon Sep 17 00:00:00 2001 From: moraxu Date: Sat, 17 May 2025 04:14:09 +0000 Subject: [PATCH 4/6] Skip FP8 test in CLI Signed-off-by: moraxu --- tests/integration/defs/accuracy/test_cli_flow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index ab8532c3f49c..4d24967c95ab 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -210,6 +210,8 @@ class TestNemotronSuper(CliFlowAccuracyTestHarness): def test_auto_dtype_tp2(self): self.run(tasks=[MMLU(self.MODEL_NAME)], tp_size=2, dtype='auto') + @pytest.mark.skip( + reason="nemotron-nas scripts have to accommodate fp8 flags") @pytest.mark.skip_less_device(2) @pytest.mark.skip_device_not_contain(["H100", "B200"]) def test_fp8_prequantized_tp2(self, mocker): From ad3ecb65ff2956930474bc6f16c6d314024637a6 Mon Sep 17 00:00:00 2001 From: moraxu Date: Tue, 20 May 2025 20:37:52 -0700 Subject: [PATCH 5/6] Address reviews Signed-off-by: moraxu --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 2 +- tests/integration/test_lists/qa/llm_sanity_test.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 97a560e0c6ad..3156782e977c 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -891,7 +891,7 @@ def test_auto_dtype_tp8(self): task.evaluate(llm) -class TestNemotronSuper(LlmapiAccuracyTestHarness): +class TestLlama3_3NemotronSuper49Bv1(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Llama-3_3-Nemotron-Super-49B-v1" MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1" diff --git a/tests/integration/test_lists/qa/llm_sanity_test.txt b/tests/integration/test_lists/qa/llm_sanity_test.txt index d5cc10f8a8e2..e79a38e73ed5 100644 --- a/tests/integration/test_lists/qa/llm_sanity_test.txt +++ b/tests/integration/test_lists/qa/llm_sanity_test.txt @@ -136,8 +136,8 @@ accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-c accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2 accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 -accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 -accuracy/test_cli_flow.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] From 6f39ca9fadc2e7e13c79fef0a3f1e9e8720ca2a1 Mon Sep 17 00:00:00 2001 From: moraxu Date: Fri, 23 May 2025 00:42:27 +0000 Subject: [PATCH 6/6] Address review comments Signed-off-by: moraxu --- tests/integration/defs/accuracy/test_cli_flow.py | 2 +- tests/integration/test_lists/qa/examples_test_list.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index 4d24967c95ab..2ba3cf657dc3 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -201,7 +201,7 @@ def test_fp8_prequantized(self, mocker): # TODO: Remove the CLI tests once NIMs use PyTorch backend -class TestNemotronSuper(CliFlowAccuracyTestHarness): +class TestLlama3_3NemotronSuper49Bv1(CliFlowAccuracyTestHarness): MODEL_NAME = "nvidia/Llama-3_3-Nemotron-Super-49B-v1" MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1" EXAMPLE_FOLDER = "models/core/nemotron_nas" diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index d401068daa29..b6f31c083cd5 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -445,10 +445,10 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp_ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 -accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 -accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_fp8_prequantized_tp2 -accuracy/test_cli_flow.py::TestNemotronSuper::test_auto_dtype_tp2 -accuracy/test_cli_flow.py::TestNemotronSuper::test_fp8_prequantized_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_fp8_prequantized_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_fp8_prequantized_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype