diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index d9c9980d5e61..1ff754f97342 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -434,6 +434,9 @@ def convert(self): f"--dtype={self.dtype}", ] + if "nemotron_nas" in self.EXAMPLE_FOLDER: + convert_cmd.append("--trust_remote_code") + if self.MODEL_FORMAT == "NEMO": convert_cmd.append(f"--nemo_ckpt_path={self.MODEL_PATH}") else: diff --git a/tests/integration/defs/accuracy/references/gpqa_diamond.yaml b/tests/integration/defs/accuracy/references/gpqa_diamond.yaml index 3fccc693227e..9e4e988f335d 100644 --- a/tests/integration/defs/accuracy/references/gpqa_diamond.yaml +++ b/tests/integration/defs/accuracy/references/gpqa_diamond.yaml @@ -16,3 +16,5 @@ deepseek-ai/DeepSeek-R1: accuracy: 70.45 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 44.95 + - quant_algo: FP8 + accuracy: 49.49 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index a19abf12e8d3..9748b122bcd1 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -65,5 +65,7 @@ Qwen3/Qwen3-30B-A3B: accuracy: 83.43 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 92.57 + - quant_algo: FP8 + accuracy: 92.42 nvidia/Nemotron-H-8B-Base-8K: - accuracy: 46.20 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 51c1cdd69ea1..4380b799764b 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -121,6 +121,8 @@ Qwen3/Qwen3-30B-A3B: accuracy: 80.65 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 79.43 + - quant_algo: FP8 + accuracy: 79.26 nvidia/Llama-3.1-Nemotron-Nano-8B-v1: - accuracy: 57.97 nvidia/Nemotron-H-8B-Base-8K: diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index e9fdac5dcbec..2ba3cf657dc3 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -200,6 +200,30 @@ def test_fp8_prequantized(self, mocker): self.run(quant_algo=QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) +# TODO: Remove the CLI tests once NIMs use PyTorch backend +class TestLlama3_3NemotronSuper49Bv1(CliFlowAccuracyTestHarness): + MODEL_NAME = "nvidia/Llama-3_3-Nemotron-Super-49B-v1" + MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1" + EXAMPLE_FOLDER = "models/core/nemotron_nas" + + @pytest.mark.skip_less_device(2) + def test_auto_dtype_tp2(self): + self.run(tasks=[MMLU(self.MODEL_NAME)], tp_size=2, dtype='auto') + + @pytest.mark.skip( + reason="nemotron-nas scripts have to accommodate fp8 flags") + @pytest.mark.skip_less_device(2) + @pytest.mark.skip_device_not_contain(["H100", "B200"]) + def test_fp8_prequantized_tp2(self, mocker): + mocker.patch.object( + self.__class__, "MODEL_PATH", + f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8" + ) + self.run(tasks=[MMLU(self.MODEL_NAME)], + tp_size=2, + quant_algo=QuantAlgo.FP8) + + class TestPhi2(CliFlowAccuracyTestHarness): MODEL_NAME = "microsoft/phi-2" MODEL_PATH = f"{llm_models_root()}/phi-2" diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index ad8b59eec5c8..3156782e977c 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -891,7 +891,7 @@ def test_auto_dtype_tp8(self): task.evaluate(llm) -class TestNemotronSuper(LlmapiAccuracyTestHarness): +class TestLlama3_3NemotronSuper49Bv1(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Llama-3_3-Nemotron-Super-49B-v1" MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1" @@ -906,6 +906,20 @@ def test_auto_dtype_tp2(self): task.evaluate(llm, extra_evaluator_kwargs=dict(apply_chat_template=True)) + @pytest.mark.skip_less_device(2) + @pytest.mark.skip_device_not_contain(["H100", "B200"]) + def test_fp8_prequantized_tp2(self): + model_path = f"{llm_models_root()}/nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8" + with LLM(model_path, tensor_parallel_size=2) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + task = GPQADiamond(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=dict(apply_chat_template=True)) + class TestNemotronNano(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Llama-3.1-Nemotron-Nano-8B-v1" diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index 6dd0b6c834f2..b6f31c083cd5 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -445,7 +445,10 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp_ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 -accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_fp8_prequantized_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_fp8_prequantized_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype diff --git a/tests/integration/test_lists/qa/llm_sanity_test.txt b/tests/integration/test_lists/qa/llm_sanity_test.txt index d174a0a1cfc6..e79a38e73ed5 100644 --- a/tests/integration/test_lists/qa/llm_sanity_test.txt +++ b/tests/integration/test_lists/qa/llm_sanity_test.txt @@ -136,7 +136,8 @@ accuracy/test_llm_api_pytorch.py::TestLlama4ScoutInstruct::test_auto_dtype[tp8-c accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2 accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8 -accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2 +accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 +accuracy/test_cli_flow.py::TestLlama3_3NemotronSuper49Bv1::test_auto_dtype_tp2 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]