From 1cc3242b9f81e04ab1669a6a0dba6db3df9477ef Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:55:36 -0700 Subject: [PATCH 1/4] debug ci Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- jenkins/L0_Test.groovy | 3 +- tests/integration/defs/test_unittests.py | 2 +- .../test_lists/test-db/l0_b200.yml | 46 ------------------- .../_torch/thop/parallel/test_fp8_linear.py | 3 ++ 4 files changed, 5 insertions(+), 49 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index f40863560276..b7c89084f890 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1989,8 +1989,7 @@ def launchTestJobs(pipeline, testFilter) "H100_PCIe-CPP-2": ["h100-cr", "l0_h100", 2, 2], "H100_PCIe-TensorRT-1": ["h100-cr", "l0_h100", 1, 2], "H100_PCIe-TensorRT-2": ["h100-cr", "l0_h100", 2, 2], - "B200_PCIe-PyTorch-1": ["b100-ts2", "l0_b200", 1, 2], - "B200_PCIe-PyTorch-2": ["b100-ts2", "l0_b200", 2, 2], + "B200_PCIe-PyTorch-1": ["b100-ts2", "l0_b200", 1, 1], "RTX5090-PyTorch-1": ["rtx-5090", "l0_gb202", 1, 1], "RTX5080-TensorRT-1": ["rtx-5080", "l0_gb203", 1, 2], "RTX5080-TensorRT-2": ["rtx-5080", "l0_gb203", 2, 2], diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index 3f13ed10b7be..abdf17d31430 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -123,7 +123,7 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): f'results-sub-unittests-{case_fn}.xml') command = [ - '-m', 'pytest', ignore_opt, "-v", "--timeout=2400", + '-m', 'pytest', ignore_opt, "-s", "-v", "--timeout=2400", "--timeout-method=thread" ] if test_prefix: diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 031548955b46..9975097dbfc6 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -14,53 +14,7 @@ l0_b200: backend: pytorch tests: # ------------- PyTorch tests --------------- - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_4] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_64] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[attn_backend=TRTLLM-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[attn_backend=TRTLLM-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=nvfp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[True-True-cutlass] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[True-True-trtllm] - - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[True-True-triton] - - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] # nvbugs 5505402 - - disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] # nvbugs 5300551 - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] - - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] - - test_e2e.py::test_ptp_quickstart_advanced_mixed_precision - - test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] - - test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct] - - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] - - unittest/_torch/attention - - unittest/_torch/compilation - - unittest/_torch/debugger - - unittest/_torch/executor - - unittest/_torch/misc - - unittest/_torch/modules - - unittest/_torch/multimodal - - unittest/_torch/sampler - - unittest/_torch/speculative - unittest/_torch/thop/parallel - - unittest/_torch/thop/serial - - unittest/_torch/modeling -k "modeling_llama" - - unittest/_torch/modeling -k "modeling_mixtral" - - unittest/_torch/modeling -k "modeling_gpt_oss" - - unittest/_torch/auto_deploy/unit/singlegpu -k "not test_trtllm_bench_backend_comparison" - condition: ranges: system_gpu_count: diff --git a/tests/unittest/_torch/thop/parallel/test_fp8_linear.py b/tests/unittest/_torch/thop/parallel/test_fp8_linear.py index bd65c2f272ad..12560d4351f9 100644 --- a/tests/unittest/_torch/thop/parallel/test_fp8_linear.py +++ b/tests/unittest/_torch/thop/parallel/test_fp8_linear.py @@ -11,6 +11,9 @@ @skip_pre_hopper @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) def test_fp8_linear(dtype): + print( + f"TORCHINDUCTOR_COMPILE_THREADS is: {os.environ.get('TORCHINDUCTOR_COMPILE_THREADS')}" + ) SEQ_LEN = 10 HIDDEN_SIZE = 128 torch.manual_seed(0) From 5fd97c9fe1c4da104cea8ce2be897f1f39684b9b Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:17:06 -0700 Subject: [PATCH 2/4] debug Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index b7c89084f890..491e0c7a0f8a 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -655,7 +655,7 @@ def testFilter = [ (ONLY_ONE_GROUP_CHANGED): "", (DEBUG_MODE): false, (AUTO_TRIGGER_TAG_LIST): [], - (DETAILED_LOG): false, + (DETAILED_LOG): true, ] @Field From d9fe537f0f741ea0bb65fda3d1698dae51ea37b7 Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Mon, 8 Sep 2025 14:02:21 -0700 Subject: [PATCH 3/4] debug ci Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- jenkins/L0_Test.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 491e0c7a0f8a..79d4ca5d96fa 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1627,6 +1627,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO extraInternalEnv, "pytest", "-v", + "-s", testFilter[(DETAILED_LOG)] ? "-s" : "", "--timeout-method=thread", "--apply-test-list-correction", From 6c7be6be1af8dc8345e2b9a30e841ced62a657dc Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Mon, 8 Sep 2025 15:31:53 -0700 Subject: [PATCH 4/4] debug ci Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/unittest/_torch/thop/parallel/test_fp8_linear.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unittest/_torch/thop/parallel/test_fp8_linear.py b/tests/unittest/_torch/thop/parallel/test_fp8_linear.py index 12560d4351f9..f6fa5a1f2110 100644 --- a/tests/unittest/_torch/thop/parallel/test_fp8_linear.py +++ b/tests/unittest/_torch/thop/parallel/test_fp8_linear.py @@ -11,9 +11,11 @@ @skip_pre_hopper @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) def test_fp8_linear(dtype): - print( - f"TORCHINDUCTOR_COMPILE_THREADS is: {os.environ.get('TORCHINDUCTOR_COMPILE_THREADS')}" - ) + t = os.environ.get('TORCHINDUCTOR_COMPILE_THREADS') + + assert t is not None, "TORCHINDUCTOR_COMPILE_THREADS is not set" + assert int(t) == 1, "TORCHINDUCTOR_COMPILE_THREADS is not set to 1" + SEQ_LEN = 10 HIDDEN_SIZE = 128 torch.manual_seed(0)