From b6a269d75b527e906324b3c53ea3f1ce14e7e8f4 Mon Sep 17 00:00:00 2001 From: qqiao Date: Wed, 11 Jun 2025 03:00:00 -0700 Subject: [PATCH 01/16] Add rtx pro 6000 post-merge tests for DS Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 1 + .../test_lists/test-db/l0_rtx_pro_6000_1.yml | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e99081d1c8c8..eb5ed3fa7182 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1843,6 +1843,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 1, 1, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml new file mode 100644 index 000000000000..02ceb36e4fb9 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml @@ -0,0 +1,74 @@ +version: 0.0.1 +l0_rtx_pro_6000_1: +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*6000*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: post_merge + backend: pytorch + tests: + - unittest/_torch/multi_gpu_modeling -k "deepseek" + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one_mtp[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_attention_dp_overlap_one_mtp[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap_cuda_graph[DeepSeek-V3-Lite-fp8] + # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_overlap_cuda_graph[DeepSeek-V3-Lite-fp8] From d648dd321a43e56534fba1510d48c45baca95905 Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 12 Jun 2025 00:20:14 -0700 Subject: [PATCH 02/16] Set open driver for rtx-pro-6000 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index eb5ed3fa7182..bc5af80eb99d 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1,4 +1,4 @@ -@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ +@Library(['bloom-jenkins-shared-lib@emma/add_open_driver', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException import groovy.transform.Field @@ -643,7 +643,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_OPEN_NVIDIA_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. From c7cebc05700157c81379a6424e1db960416910d2 Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 12 Jun 2025 19:58:55 -0700 Subject: [PATCH 03/16] Update skip condition and split to 2 stage Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index bc5af80eb99d..f203cd5a9878 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1844,6 +1844,8 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 1, 1, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 1, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 2, 2, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { From 137290d2b3e3793e20571935636669cddbcc9b14 Mon Sep 17 00:00:00 2001 From: qqiao Date: Mon, 16 Jun 2025 00:48:57 -0700 Subject: [PATCH 04/16] Add more ds cases on rtx-pro-6000 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 5 +- .../test_lists/test-db/l0_rtx_pro_6000_1.yml | 74 ------------------- 2 files changed, 2 insertions(+), 77 deletions(-) delete mode 100644 tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index f203cd5a9878..ea5d9cfdc1bc 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1843,9 +1843,8 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 1, 1, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 1, 2, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-[Post-Merge]-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000_1", 2, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml deleted file mode 100644 index 02ceb36e4fb9..000000000000 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000_1.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 0.0.1 -l0_rtx_pro_6000_1: -- condition: - ranges: - system_gpu_count: - gte: 4 - lte: 4 - wildcards: - gpu: - - '*6000*' - linux_distribution_name: ubuntu* - cpu: x86_64 - terms: - stage: post_merge - backend: pytorch - tests: - - unittest/_torch/multi_gpu_modeling -k "deepseek" - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one_mtp[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_attention_dp_overlap_one_mtp[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap_cuda_graph[DeepSeek-V3-Lite-fp8] - # - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_overlap_cuda_graph[DeepSeek-V3-Lite-fp8] From 70dcc2102af18f6d8e1bfbbd57c323050bea4db1 Mon Sep 17 00:00:00 2001 From: qqiao Date: Tue, 17 Jun 2025 01:09:35 -0700 Subject: [PATCH 05/16] Waive the hang cases first Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ea5d9cfdc1bc..229b58d78ce4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1,4 +1,4 @@ -@Library(['bloom-jenkins-shared-lib@emma/add_open_driver', 'trtllm-jenkins-shared-lib@main']) _ +@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException import groovy.transform.Field @@ -643,7 +643,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_OPEN_NVIDIA_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. From a9cdf7f7309ba03d900aedf754c6a4294ded9d44 Mon Sep 17 00:00:00 2001 From: qqiao Date: Tue, 29 Jul 2025 01:27:18 -0700 Subject: [PATCH 06/16] Test for latest open driver Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 229b58d78ce4..b8a0830dd42e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,7 +643,8 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + // def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? "580.50-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. From e1e2421ec87d8a126c3033cd5a2cbfb41bdc0fa5 Mon Sep 17 00:00:00 2001 From: qqiao Date: Tue, 29 Jul 2025 01:29:16 -0700 Subject: [PATCH 07/16] Run all the tests on 0082 which seems stable Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index b8a0830dd42e..9e3785c91570 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1844,8 +1844,8 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 1, 4], + //"RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { From c221e78fdd5a9a5b2c00128fb9e7ddb1ae4ed03c Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 14 Aug 2025 05:33:33 -0700 Subject: [PATCH 08/16] Add single gpu tests for rtxpro6000 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 8 ++++---- tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9e3785c91570..e8a932cab8d7 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,8 +643,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - // def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION - def driverVersion = type.contains("rtx-pro-6000") ? "580.50-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. @@ -1801,6 +1800,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "RTX5090-PyTorch-1": ["rtx-5090", "l0_gb202", 1, 1], "RTX5080-TensorRT-1": ["rtx-5080", "l0_gb203", 1, 2], "RTX5080-TensorRT-2": ["rtx-5080", "l0_gb203", 2, 2], + "RTXPro6000-Pytorch-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], // Currently post-merge test stages only run tests with "stage: post_merge" mako // in the test-db. This behavior may change in the future. "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 1], @@ -1844,8 +1844,8 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 1, 4], - //"RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 49c29329dc72..72b85b527e87 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -10,7 +10,7 @@ l0_rtx_pro_6000: - '*6000*' linux_distribution_name: ubuntu* terms: - stage: post_merge + stage: pre_merge backend: pytorch tests: # ------------- PyTorch tests --------------- From 30bd4a22077602655f558821931e48c6f7200fce Mon Sep 17 00:00:00 2001 From: qqiao Date: Sat, 16 Aug 2025 02:44:28 -0700 Subject: [PATCH 09/16] Update for comments Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e8a932cab8d7..befd641dfd21 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,7 +643,8 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = type.contains("rtx-pro-6000") ? Constants.DEFAULT_NVIDIA_OPEN_DRIVER_VERSION : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + //def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? "580.50-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. @@ -1844,8 +1845,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], - "RTXPro6000-4_GPUs-Pytorch-DeepSeek-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 1, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { @@ -1864,7 +1864,6 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) fullSet = parallelJobs.keySet() x86SlurmTestConfigs = [ - "RTXPro6000-PyTorch-Post-Merge-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], "DGX_B200-4_GPUs-PyTorch-Post-Merge-1": ["b200-x4", "l0_dgx_b200", 1, 1, 4], ] fullSet += x86SlurmTestConfigs.keySet() From 975381a3c7aab8e263e411590c4dcd64b9f71677 Mon Sep 17 00:00:00 2001 From: qqiao Date: Sun, 17 Aug 2025 23:48:39 -0700 Subject: [PATCH 10/16] Fix multi-gpu for rtxpro6000 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index befd641dfd21..e9da72781615 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,8 +643,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - //def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION - def driverVersion = type.contains("rtx-pro-6000") ? "580.50-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. @@ -662,7 +661,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod targetCould = "kubernetes" // The following GPU types doesn't support dynamic driver flashing. - if (type.contains("dgx-h100") || type.contains("dgx-h200") || type in ["b100-ts2", "gh200", "rtx-5080", "rtx-5090"]) { + if (type.contains("dgx-h100") || type.contains("dgx-h200") || type.contains("rtx-pro-6000") || type in ["b100-ts2", "gh200", "rtx-5080", "rtx-5090"]) { selectors = """ kubernetes.io/arch: ${arch} kubernetes.io/os: linux @@ -2458,7 +2457,7 @@ pipeline { def testPhase2StageName = env.testPhase2StageName if (testPhase2StageName) { - def dgxSigns = ["DGX_H100", "DGX_H200", "GB200", "DGX_B200"] + def dgxSigns = ["DGX_H100", "DGX_H200", "GB200", "DGX_B200", "RTXPro6000-4_GPUs"] singleGpuJobs = parallelJobs.findAll{!dgxSigns.any{sign -> it.key.contains(sign)}} dgxJobs = parallelJobs.findAll{dgxSigns.any{sign -> it.key.contains(sign)}} } From 62cd3102e52441d20c651545097bfa1dbb4ba396 Mon Sep 17 00:00:00 2001 From: qqiao Date: Tue, 19 Aug 2025 01:52:10 -0700 Subject: [PATCH 11/16] Test for node 081 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e9da72781615..4e859f089cea 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -665,6 +665,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod selectors = """ kubernetes.io/arch: ${arch} kubernetes.io/os: linux + kubernetes.io/hostname: smc521ge-0081.ipp2a2.colossus nvidia.com/gpu_type: ${gpuType}""" } else if (perfMode && !hasMultipleGPUs) { // Not using the "perf" node currently due to hardware resource constraint. From 41f4656a38243b17db201e0e6adc16c5712d7df2 Mon Sep 17 00:00:00 2001 From: qqiao Date: Tue, 19 Aug 2025 22:34:44 -0700 Subject: [PATCH 12/16] Split the tests stage to test Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 4e859f089cea..02d4cd065f54 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1845,7 +1845,8 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], - "RTXPro6000-4_GPUs-Pytorch-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 1, 4], + "RTXPro6000-4_GPUs-Pytorch-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], + "RTXPro6000-4_GPUs-Pytorch-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], ] parallelJobs = x86TestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "amd64", values[4] ?: 1, key.contains("Perf")), { From 823ebe1b00423fcca55db21e9a7ab2ce2bd85752 Mon Sep 17 00:00:00 2001 From: qqiao Date: Wed, 20 Aug 2025 01:01:47 -0700 Subject: [PATCH 13/16] Update cpu and mem for multi-gpu pod Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 02d4cd065f54..9dce9bd659ae 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -647,7 +647,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. - if ((type.contains("dgx-h100") || type.contains("dgx-h200")) && hasMultipleGPUs) + if (hasMultipleGPUs) { // Not a hard requirement, but based on empirical values. memorySize = "${gpuCount * 150}" + "Gi" From 589b713b8cefb83d0a82d82644816e7669799318 Mon Sep 17 00:00:00 2001 From: qqiao Date: Wed, 20 Aug 2025 23:37:50 -0700 Subject: [PATCH 14/16] Test latest open driver for rtxpro6000 Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9dce9bd659ae..2b2710819731 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,7 +643,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = type.contains("rtx-pro-6000") ? "580.65.06-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. @@ -665,7 +665,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod selectors = """ kubernetes.io/arch: ${arch} kubernetes.io/os: linux - kubernetes.io/hostname: smc521ge-0081.ipp2a2.colossus + nvidia.com/driver_version: '${driverVersion}' nvidia.com/gpu_type: ${gpuType}""" } else if (perfMode && !hasMultipleGPUs) { // Not using the "perf" node currently due to hardware resource constraint. @@ -1270,7 +1270,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO sh "nproc && free -g && hostname" echoNodeAndGpuInfo(pipeline, stageName) sh "cat ${MODEL_CACHE_DIR}/README" - sh "nvidia-smi -q" + sh "nvidia-smi topo -m" sh "df -h" // setup HF_HOME to cache model and datasets @@ -1801,7 +1801,6 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "RTX5090-PyTorch-1": ["rtx-5090", "l0_gb202", 1, 1], "RTX5080-TensorRT-1": ["rtx-5080", "l0_gb203", 1, 2], "RTX5080-TensorRT-2": ["rtx-5080", "l0_gb203", 2, 2], - "RTXPro6000-Pytorch-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], // Currently post-merge test stages only run tests with "stage: post_merge" mako // in the test-db. This behavior may change in the future. "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 1], @@ -1845,6 +1844,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], + "RTXPro6000-Pytorch-Post-Merge-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], "RTXPro6000-4_GPUs-Pytorch-Post-Merge-1": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 1, 2, 4], "RTXPro6000-4_GPUs-Pytorch-Post-Merge-2": ["rtx-pro-6000-x4", "l0_rtx_pro_6000", 2, 2, 4], ] From b89e9444aa6ce83373fb5172ab412fdf8c32c6ec Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 21 Aug 2025 02:57:14 -0700 Subject: [PATCH 15/16] move single gpu test to post-merge Signed-off-by: qqiao --- tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 72b85b527e87..49c29329dc72 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -10,7 +10,7 @@ l0_rtx_pro_6000: - '*6000*' linux_distribution_name: ubuntu* terms: - stage: pre_merge + stage: post_merge backend: pytorch tests: # ------------- PyTorch tests --------------- From 4515b1451b26d08ef01bda527112bf824bb3fc5a Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 21 Aug 2025 19:25:04 -0700 Subject: [PATCH 16/16] Remove debug info Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 2b2710819731..28c5c182fb45 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -643,10 +643,9 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def hasMultipleGPUs = (gpuCount > 1) def memorySize = "${TESTER_MEMORY}" def storageSize = "300Gi" - def driverVersion = type.contains("rtx-pro-6000") ? "580.65.06-open" : Constants.DEFAULT_NVIDIA_DRIVER_VERSION + def driverVersion = Constants.DEFAULT_NVIDIA_DRIVER_VERSION def cpuCount = "${TESTER_CORES}" - // Multi-GPU only supports DGX-H100 and DGX-H200 due to the hardware stability. if (hasMultipleGPUs) { // Not a hard requirement, but based on empirical values. @@ -665,7 +664,6 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod selectors = """ kubernetes.io/arch: ${arch} kubernetes.io/os: linux - nvidia.com/driver_version: '${driverVersion}' nvidia.com/gpu_type: ${gpuType}""" } else if (perfMode && !hasMultipleGPUs) { // Not using the "perf" node currently due to hardware resource constraint. @@ -1270,6 +1268,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO sh "nproc && free -g && hostname" echoNodeAndGpuInfo(pipeline, stageName) sh "cat ${MODEL_CACHE_DIR}/README" + sh "nvidia-smi -q" sh "nvidia-smi topo -m" sh "df -h"