From 2fc1c3d828b87d7f7782d0d40f907b0d649d612d Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 19 Aug 2025 11:52:24 +0800 Subject: [PATCH 1/5] add llmapi trt flow test case with cuda graph and generate logits Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api.py | 14 +++++++++++++- .../test_lists/qa/llm_function_full.txt | 1 + tests/integration/test_lists/waives.txt | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index 321591228bdb..fd031e655349 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -15,7 +15,8 @@ import pytest from tensorrt_llm._tensorrt_engine import LLM -from tensorrt_llm.llmapi import EagleDecodingConfig, KvCacheConfig +from tensorrt_llm.llmapi import (EagleDecodingConfig, + ExtendedRuntimePerfKnobConfig, KvCacheConfig) from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization import QuantAlgo @@ -76,6 +77,17 @@ def test_guided_decoding_4gpus(self, backend: str): task = JsonModeEval(self.MODEL_NAME) task.evaluate(llm) + def test_gather_generation_logits_cuda_graph(self): + extended_runtime_perf_knob_config = ExtendedRuntimePerfKnobConfig( + cuda_graph_mode=True, cuda_graph_cache_size=1) + llm = LLM( + self.MODEL_PATH, + gather_generation_logits=True, + extended_runtime_perf_knob_config=extended_runtime_perf_knob_config) + with llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + class TestLlama3_2_1B(LlmapiAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-3.2-1B" diff --git a/tests/integration/test_lists/qa/llm_function_full.txt b/tests/integration/test_lists/qa/llm_function_full.txt index f293321b57cb..b671920318d1 100644 --- a/tests/integration/test_lists/qa/llm_function_full.txt +++ b/tests/integration/test_lists/qa/llm_function_full.txt @@ -420,6 +420,7 @@ accuracy/test_cli_flow.py::TestQwen2_57B_A14B::test_tp4 accuracy/test_cli_flow.py::TestQwen2_57B_A14B::test_tp2pp2 accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] +accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 5fbaae601930..85f0fe58a77a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -314,3 +314,5 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency] test_e2e.py::test_ptp_quickstart_multimodal[mistral-small-3.1-24b-instruct-Mistral-Small-3.1-24B-Instruct-2503-image-True] SKIP (https://nvbugs/5459817) llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp SKIP (https://nvbugs/5461796) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/5459811) +accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_eagle3] SKIP (https://nvbugs/5437384) +accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph SKIP (https://nvbugs/5365525) From 21bd53811a3bea3e6f901212e9a6c5894474e5bb Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 21 Jul 2025 16:20:43 +0800 Subject: [PATCH 2/5] add llmapi logprobs test with trt flow Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../defs/accuracy/references/cnn_dailymail.yaml | 2 ++ tests/integration/defs/accuracy/test_llm_api.py | 12 +++++++++++- .../integration/test_lists/qa/llm_function_full.txt | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index 5d5adcbf9b4a..af6621d7df36 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -132,6 +132,8 @@ meta-llama/Llama-3.1-8B-Instruct: - accuracy: 33.640 - spec_dec_algo: Eagle accuracy: 33.640 + - extra_acc_spec: logprobs=2 + accuracy: 30.522 - quant_algo: FP8 accuracy: 33.841 - quant_algo: FP8 diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index fd031e655349..5b8ba65345f7 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -16,7 +16,8 @@ from tensorrt_llm._tensorrt_engine import LLM from tensorrt_llm.llmapi import (EagleDecodingConfig, - ExtendedRuntimePerfKnobConfig, KvCacheConfig) + ExtendedRuntimePerfKnobConfig, KvCacheConfig, + SamplingParams) from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization import QuantAlgo @@ -88,6 +89,15 @@ def test_gather_generation_logits_cuda_graph(self): task = CnnDailymail(self.MODEL_NAME) task.evaluate(llm) + def test_logprobs(self): + sampling_config = SamplingParams(logprobs=2) + llm = LLM(self.MODEL_PATH, gather_generation_logits=True) + with llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm, + sampling_params=sampling_config, + extra_acc_spec="logprobs=2") + class TestLlama3_2_1B(LlmapiAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-3.2-1B" diff --git a/tests/integration/test_lists/qa/llm_function_full.txt b/tests/integration/test_lists/qa/llm_function_full.txt index b671920318d1..181a38b954b4 100644 --- a/tests/integration/test_lists/qa/llm_function_full.txt +++ b/tests/integration/test_lists/qa/llm_function_full.txt @@ -421,6 +421,7 @@ accuracy/test_cli_flow.py::TestQwen2_57B_A14B::test_tp2pp2 accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph +accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_logprobs accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise From d60ba6368a5d3533f662ec04a73fa593f81dd0a4 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 22 Jul 2025 14:21:00 +0800 Subject: [PATCH 3/5] add llmapi trt flow phi-4-mini-instruct acc test Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../accuracy/references/cnn_dailymail.yaml | 4 ++++ .../defs/accuracy/references/mmlu.yaml | 2 ++ .../integration/defs/accuracy/test_llm_api.py | 20 +++++++++++++++++++ .../test_lists/qa/llm_function_full.txt | 2 ++ 4 files changed, 28 insertions(+) diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index af6621d7df36..03416943ce83 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -240,6 +240,10 @@ mistralai/Mixtral-8x22B-v0.1: accuracy: 25.519 - quant_algo: W8A16 accuracy: 27.427 +microsoft/Phi-4-mini-instruct: + - accuracy: 32.958 + - quant_algo: FP8 + accuracy: 32.823 google/gemma-2b: - accuracy: 23.194 - quant_algo: W8A16 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 542725fc4254..27d338aa32bb 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -232,6 +232,8 @@ nvidia/Nemotron-H-56B-Base-8K: accuracy: 83.82 microsoft/Phi-4-mini-instruct: - accuracy: 68.98 + - quant_algo: FP8 + accuracy: 68.30 bigcode/starcoder2-7b: - accuracy: 41.35 - quant_algo: FP8 diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index 5b8ba65345f7..5de4ca2c1384 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -266,6 +266,26 @@ def test_awq_tp2(self): task.evaluate(llm) +class TestPhi4MiniInstruct(LlmapiAccuracyTestHarness): + MODEL_NAME = "microsoft/Phi-4-mini-instruct" + MODEL_PATH = f"{llm_models_root()}/Phi-4-mini-instruct" + + def test_auto_dtype(self): + with LLM(self.MODEL_PATH) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + def test_fp8(self): + quant_config = QuantConfig(QuantAlgo.FP8) + with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + class TestQwen2_7BInstruct(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen2-7B-Instruct" MODEL_PATH = f"{llm_models_root()}/Qwen2-7B-Instruct" diff --git a/tests/integration/test_lists/qa/llm_function_full.txt b/tests/integration/test_lists/qa/llm_function_full.txt index 181a38b954b4..b3b4a496728e 100644 --- a/tests/integration/test_lists/qa/llm_function_full.txt +++ b/tests/integration/test_lists/qa/llm_function_full.txt @@ -422,6 +422,8 @@ accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_gather_generation_logits_cuda_graph accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_logprobs +accuracy/test_llm_api.py::TestPhi4MiniInstruct::test_auto_dtype +accuracy/test_llm_api.py::TestPhi4MiniInstruct::test_fp8 accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_auto_dtype accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_weight_only accuracy/test_llm_api.py::TestLlama3_1_8B::test_fp8_rowwise From af8412ddea33bbf3406c1bec9588e77db0446444 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:00:29 +0800 Subject: [PATCH 4/5] add nemo 12b base test cases Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../accuracy/references/cnn_dailymail.yaml | 9 ++-- .../defs/accuracy/references/mmlu.yaml | 4 ++ .../integration/defs/accuracy/test_llm_api.py | 43 ++++++++++++++++--- .../defs/accuracy/test_llm_api_pytorch.py | 30 +++++++++++++ .../test_lists/qa/llm_function_full.txt | 6 ++- 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index 03416943ce83..0c469f2d9442 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -45,6 +45,8 @@ microsoft/Phi-3.5-mini-instruct: - accuracy: 31.354 microsoft/Phi-4-mini-instruct: - accuracy: 32.921 + - quant_algo: FP8 + accuracy: 32.823 bigcode/starcoder2-7b: - accuracy: 26.611 - quant_algo: FP8 @@ -209,7 +211,8 @@ mistralai/Mistral-7B-Instruct-v0.3: accuracy: 31.201 mistralai/Mistral-Small-3.1-24B-Instruct-2503: - accuracy: 29.20 -mistralai/Mistral-Nemo-Base-2407: +mistralai/Mistral-Nemo-12b-Base: + - accuracy: 28.906 - quant_algo: FP8 kv_cache_quant_algo: FP8 accuracy: 24.0 @@ -240,10 +243,6 @@ mistralai/Mixtral-8x22B-v0.1: accuracy: 25.519 - quant_algo: W8A16 accuracy: 27.427 -microsoft/Phi-4-mini-instruct: - - accuracy: 32.958 - - quant_algo: FP8 - accuracy: 32.823 google/gemma-2b: - accuracy: 23.194 - quant_algo: W8A16 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 27d338aa32bb..ee7cc99c40dd 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -277,3 +277,7 @@ GPT-OSS/MXFP4: accuracy: 75.50 - quant_algo: W4A8_MXFP4_FP8 accuracy: 75.50 +mistralai/Mistral-Nemo-12b-Base: + - accuracy: 69.66 + - quant_algo: FP8 + accuracy: 69.66 diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index 5de4ca2c1384..8f9761151976 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -79,6 +79,7 @@ def test_guided_decoding_4gpus(self, backend: str): task.evaluate(llm) def test_gather_generation_logits_cuda_graph(self): + """RCCA: https://nvbugs/5365525""" extended_runtime_perf_knob_config = ExtendedRuntimePerfKnobConfig( cuda_graph_mode=True, cuda_graph_cache_size=1) llm = LLM( @@ -199,18 +200,49 @@ def test_quant_tp4(self, quant): task.evaluate(llm) -class TestMistral_Nemo_12B_Base(LlmapiAccuracyTestHarness): - MODEL_NAME = "mistralai/Mistral-Nemo-Base-2407" +class TestMistralNemo12B(LlmapiAccuracyTestHarness): + MODEL_NAME = "mistralai/Mistral-Nemo-12b-Base" MODEL_PATH = f"{llm_models_root()}/Mistral-Nemo-Base-2407" + @pytest.mark.skip_less_device_memory(80000) + def test_auto_dtype(self): + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9) + + with LLM(self.MODEL_PATH, + kv_cache_config=kv_cache_config, + max_batch_size=8) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + def test_auto_dtype_tp2(self): + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9) + + with LLM(self.MODEL_PATH, + kv_cache_config=kv_cache_config, + tensor_parallel_size=2, + max_batch_size=8) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + @pytest.mark.skip_less_device_memory(80000) @skip_pre_ada def test_fp8(self): - quant_config = QuantConfig(quant_algo=QuantAlgo.FP8, + quant_config = QuantConfig(QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9) - with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: + with LLM(self.MODEL_PATH, + quant_config=quant_config, + kv_cache_config=kv_cache_config, + max_batch_size=8) as llm: task = CnnDailymail(self.MODEL_NAME) task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) class TestMistral_NeMo_Minitron_8B_Instruct(LlmapiAccuracyTestHarness): @@ -277,6 +309,7 @@ def test_auto_dtype(self): task = MMLU(self.MODEL_NAME) task.evaluate(llm) + @skip_pre_ada def test_fp8(self): quant_config = QuantConfig(QuantAlgo.FP8) with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: @@ -420,7 +453,7 @@ def test_fp8(self): @skip_pre_ada def test_fp8_kvcache(self): "RCCA: https://nvbugs/5065080" - quant_config = QuantConfig(QuantAlgo.FP8, + quant_config = QuantConfig(quant_algo=QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: task = CnnDailymail(self.MODEL_NAME) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 54e8bee62505..69ad940a6452 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1773,6 +1773,36 @@ def test_auto_dtype_tp8(self): task.evaluate(llm) +class TestMistralNemo12B(LlmapiAccuracyTestHarness): + MODEL_NAME = "mistralai/Mistral-Nemo-12b-Base" + MODEL_PATH = f"{llm_models_root()}/Mistral-Nemo-Base-2407" + + @pytest.mark.skip_less_device_memory(80000) + def test_auto_dtype(self): + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9) + + with LLM(self.MODEL_PATH, + kv_cache_config=kv_cache_config, + max_batch_size=8) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + @pytest.mark.skip_less_device(2) + def test_auto_dtype_tp2(self): + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9) + + with LLM(self.MODEL_PATH, + kv_cache_config=kv_cache_config, + tensor_parallel_size=2, + max_batch_size=8) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + + @pytest.mark.timeout(5400) @pytest.mark.skip_less_device_memory(80000) class TestLlama3_3NemotronSuper49Bv1(LlmapiAccuracyTestHarness): diff --git a/tests/integration/test_lists/qa/llm_function_full.txt b/tests/integration/test_lists/qa/llm_function_full.txt index b3b4a496728e..aa24e5b9c6d8 100644 --- a/tests/integration/test_lists/qa/llm_function_full.txt +++ b/tests/integration/test_lists/qa/llm_function_full.txt @@ -435,7 +435,9 @@ accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8_kvcache accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int4] accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int4_awq] accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int8_awq] -accuracy/test_llm_api.py::TestMistral_Nemo_12B_Base::test_fp8 +accuracy/test_llm_api.py::TestMistralNemo12B::test_auto_dtype +accuracy/test_llm_api.py::TestMistralNemo12B::test_auto_dtype_tp2 +accuracy/test_llm_api.py::TestMistralNemo12B::test_fp8 accuracy/test_llm_api.py::TestMistral_NeMo_Minitron_8B_Instruct::test_fp8 accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2 accuracy/test_llm_api.py::TestMixtral8x7B::test_smooth_quant_tp2pp2 @@ -580,6 +582,8 @@ accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestEXAONE4::test_auto_dtype accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend +accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype +accuracy/test_llm_api_pytorch.py::TestMistralNemo12B::test_auto_dtype_tp2 test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding-] test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding-] From 4200cd60735ce616f108c18704cd001ae75d3062 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 18 Aug 2025 13:32:29 +0800 Subject: [PATCH 5/5] fix invalid test name Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_h100.yml | 2 +- tests/integration/test_lists/waives.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 64f6498d095b..a52b515e6441 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -253,7 +253,7 @@ l0_h100: - examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle1] - examples/test_eagle.py::test_llm_eagle_1gpu[EAGLE-Vicuna-7B-v1.3-float16-bs1-eagle2] # 5 mins - accuracy/test_llm_api.py::TestMistral_NeMo_Minitron_8B_Instruct::test_fp8 - - accuracy/test_llm_api.py::TestMistral_Nemo_12B_Base::test_fp8 + - accuracy/test_llm_api.py::TestMistralNemo12B::test_fp8 - examples/test_multimodal.py::test_llm_multimodal_general[llava-1.5-7b-hf-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1] # 7 mins - examples/test_multimodal.py::test_llm_multimodal_general[fuyu-8b-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1] - examples/test_multimodal.py::test_llm_multimodal_general[video-neva-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False-nb:1] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 85f0fe58a77a..ed54881d3db8 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -300,7 +300,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutl accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] SKIP (https://nvbugs/5457489) accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True] SKIP (https://nvbugs/5457489) disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/5457504) -accuracy/test_llm_api.py::TestMistral_Nemo_12B_Base::test_fp8 SKIP (https://nvbugs/5413197) +accuracy/test_llm_api.py::TestMistralNemo12B::test_fp8 SKIP (https://nvbugs/5413197) triton_server/test_triton.py::test_gpt_ib_streaming[gpt-ib-streaming] SKIP (https://nvbugs/5371349) triton_server/test_triton.py::test_gpt_ib_ptuning[gpt-ib-ptuning] SKIP (https://nvbugs/5445624) triton_server/test_triton.py::test_mistral_ib_mm[mistral-ib-mm] SKIP (https://nvbugs/5371343)