From f39dd8ad108ac951740303f18379b1e2cc5c9059 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 24 Nov 2025 07:23:13 +0000 Subject: [PATCH 01/10] Generate test results when pytest timeout happens Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 53 +++++++++++++++++-- tests/integration/defs/conftest.py | 17 ++++++ .../integration/defs/utils/periodic_junit.py | 34 ++++++++++++ 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0ff8db0dcc1b..94e612c7f571 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -122,13 +122,29 @@ def uploadResults(def pipeline, SlurmCluster cluster, String nodeName, String st Utils.exec(pipeline, script: "apt-get update && apt-get install -y sshpass openssh-client") - def downloadSucceed = false + def hasTimeoutTest = false + def downloadResultSucceed = false pipeline.stage('Submit Test Results') { sh "mkdir -p ${stageName}" + // Download timeout test results + def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" + def downloadTimeoutTestCmd = "sshpass -p '${remote.passwd}' scp -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${timeoutTestFilePath} ${stageName}/" + def downloadTimeoutTestSucceed = sh(script: downloadTimeoutTestCmd, returnStatus: true) == 0 + if (downloadTimeoutTestSucceed) { + sh "ls ${stageName}" + def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") + if (timeoutTestXml != null) { + sh "echo '${timeoutTestXml}' > ${stageName}/results-timeout.xml" + hasTimeoutTest = true + } + } + // Download normal test results def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results.xml" - downloadSucceed = Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -P ${remote.port} -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${resultsFilePath} ${stageName}/", returnStatus: true, numRetries: 3) == 0 - if (downloadSucceed) { + downloadResultSucceed = Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -P ${remote.port} -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${resultsFilePath} ${stageName}/", returnStatus: true, numRetries: 3) == 0 + + echo "hasTimeoutTest: ${hasTimeoutTest}, downloadResultSucceed: ${downloadResultSucceed}" + if (hasTimeoutTest || downloadResultSucceed) { sh "ls ${stageName}" echo "Upload test results." sh "tar -czvf results-${stageName}.tar.gz ${stageName}/" @@ -142,7 +158,7 @@ def uploadResults(def pipeline, SlurmCluster cluster, String nodeName, String st } } - if (downloadSucceed) { + if (hasTimeoutTest || downloadResultSucceed) { junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") } } @@ -781,6 +797,10 @@ def getPytestBaseCommandLine( "--cov=${trtllmWheelPath}/tensorrt_llm/", "--cov-report=", "--cov-config=${coverageConfigFile}", + "--periodic-junit", + "--periodic-interval=1800", // 30 minutes + "--periodic-batch-size=1", + "--periodic-save-unfinished-test", ] if (perfMode) { @@ -1258,6 +1278,10 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu sh "mkdir -p ${stageName}" finallyRunner() if (stageIsFailed) { + def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") + if (timeoutTestXml != null) { + sh "echo '${timeoutTestXml}' > ${stageName}/results-timeout.xml" + } def stageXml = generateStageFailTestResultXml(stageName, "Stage Failed", "Stage run failed without result", "results*.xml") if (stageXml != null) { sh "echo '${stageXml}' > ${stageName}/results-stage.xml" @@ -1618,9 +1642,28 @@ def launchTestListCheck(pipeline) }) } +def generateTimeoutTestResultXml(stageName, testFilePath) { + String timeoutTests = sh(script: "cd ${stageName} && cat ${testFilePath}", returnStdout: true).trim() + echo "timeoutTests: ${timeoutTests}" + + if (timeoutTests == null || timeoutTests == "") { + return null + } + def testList = timeoutTests.split("\n") + String xmlContent = """ + """ + testList.each { test -> + xmlContent += """ + Test timeout + """ + } + xmlContent += "" + return xmlContent +} + def generateStageFailTestResultXml(stageName, subName, failureLog, resultPath) { String resultFiles = sh(script: "cd ${stageName} && ls -l ${resultPath} | wc -l", returnStdout: true).trim() - echo "${resultFiles}" + echo "resultFiles: ${resultFiles}" if (resultFiles != "0") { return null } diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index da050518ac8e..d7061350faaa 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -2147,6 +2147,15 @@ def pytest_addoption(parser): help="Enable GPU clock locking during tests. " "By default, GPU clock locking is disabled.", ) + parser.addoption( + "--periodic-save-unfinished-test", + action="store_true", + default=False, + help= + "Save unfinished test name to unfinished_test.txt during test execution (default: False). " + "This helps identify which test was running when a timeout or crash occurs. " + "Only used with --periodic-junit.", + ) @pytest.hookimpl(trylast=True) @@ -2256,6 +2265,8 @@ def pytest_configure(config): if periodic and output_dir: periodic_interval = config.getoption("--periodic-interval") periodic_batch_size = config.getoption("--periodic-batch-size") + periodic_save_unfinished_test = config.getoption( + "--periodic-save-unfinished-test", default=False) # Create output directory early (like --junitxml does) to avoid conflicts with other plugins # that may need to write to the same directory (e.g., pytest-split) @@ -2272,6 +2283,7 @@ def pytest_configure(config): 'info': print_info, 'warning': print_warning }, + save_unfinished_test=periodic_save_unfinished_test, ) # Configure and register the reporter @@ -2283,6 +2295,7 @@ def pytest_configure(config): f" Interval: {periodic_interval}s ({periodic_interval/60:.1f} min)" ) print_info(f" Batch size: {periodic_batch_size} tests") + print_info(f" Save unfinished test: {periodic_save_unfinished_test}") elif periodic and not output_dir: print_warning( "Warning: --periodic-junit requires --output-dir to be set. " @@ -2344,6 +2357,8 @@ def deselect_by_test_model_suites(test_model_suites, items, test_prefix, if periodic and output_dir: periodic_interval = config.getoption("--periodic-interval") periodic_batch_size = config.getoption("--periodic-batch-size") + periodic_save_unfinished_test = config.getoption( + "--periodic-save-unfinished-test", default=False) # Create the reporter with logger xmlpath = os.path.join(output_dir, "results.xml") @@ -2355,6 +2370,7 @@ def deselect_by_test_model_suites(test_model_suites, items, test_prefix, 'info': print_info, 'warning': print_warning }, + save_unfinished_test=periodic_save_unfinished_test, ) # Configure and register the reporter @@ -2366,6 +2382,7 @@ def deselect_by_test_model_suites(test_model_suites, items, test_prefix, f" Interval: {periodic_interval}s ({periodic_interval/60:.1f} min)" ) print_info(f" Batch size: {periodic_batch_size} tests") + print_info(f" Save unfinished test: {periodic_save_unfinished_test}") elif periodic and not output_dir: print_warning( "Warning: --periodic-junit requires --output-dir to be set. " diff --git a/tests/integration/defs/utils/periodic_junit.py b/tests/integration/defs/utils/periodic_junit.py index 7fec3eaa42ab..f7c49167a5e8 100644 --- a/tests/integration/defs/utils/periodic_junit.py +++ b/tests/integration/defs/utils/periodic_junit.py @@ -73,6 +73,7 @@ def __init__( interval: int = 18000, # Default 5 hours batch_size: int = 10, logger=None, # Optional logger (info, warning functions) + save_unfinished_test: bool = False, # Save unfinished test name in output-dir/unfinished_test.txt if True ): """ Initialize periodic reporter. @@ -85,11 +86,13 @@ def __init__( interval: Time interval in seconds between saves (default: 18000 = 5 hours) batch_size: Number of tests before triggering a save (default: 10) logger: Optional dictionary with 'info' and 'warning' functions for logging + save_unfinished_test: If True, save unfinished test name in output-dir/unfinished_test.txt """ self.xmlpath = os.path.abspath(xmlpath) self.time_interval = interval self.batch_size = batch_size self.logger = logger or {} + self.save_unfinished_test = save_unfinished_test self.completed_tests = 0 self.last_save_time = time.time() @@ -160,11 +163,42 @@ def pytest_runtest_logreport(self, report: TestReport): # Collect the report for later batch processing (fast) self.pending_reports.append(report) + output_dir = os.path.dirname(self.xmlpath) + unfinished_test_path = os.path.join(output_dir, "unfinished_test.txt") + + # save unfinished test nodeid to output-dir/unfinished_test.txt + if self.save_unfinished_test and report.when == "setup": + try: + # Create directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + with open(unfinished_test_path, "a", encoding="utf-8") as f: + f.write(report.nodeid + "\n") + except Exception as e: + self._log_warning( + f"Error writing unfinished test {report.nodeid} to {unfinished_test_path}: {e}" + ) + # Only increment counter and check for save on teardown phase if report.when == "teardown": self.completed_tests += 1 current_time = time.time() + if self.save_unfinished_test: + if os.path.exists(unfinished_test_path): + try: + with open(unfinished_test_path, "r+", + encoding="utf-8") as f: + lines = f.readlines() + f.seek(0) + f.truncate() + for line in lines: + if line.strip() != report.nodeid: + f.write(line) + except Exception as e: + self._log_warning( + f"Error clearing nodeid {report.nodeid} from {unfinished_test_path}: {e}" + ) + # Flush if batch threshold reached OR time interval elapsed should_flush_by_time = (current_time - self.last_save_time) >= self.time_interval From aa57f89dedf59da13ea51d2a57080d4b2e8093fa Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 24 Nov 2025 07:38:52 +0000 Subject: [PATCH 02/10] test code Signed-off-by: Yiqing Yan --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 3 +++ tests/integration/defs/disaggregated/test_disaggregated.py | 1 + tests/integration/defs/test_e2e.py | 2 ++ tests/integration/test_lists/test-db/l0_a10.yml | 4 ++-- tests/integration/test_lists/test-db/l0_b300.yml | 2 +- tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml | 2 +- 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index fa82a863605a..144e2abeffa4 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import time import pytest import torch @@ -51,6 +52,7 @@ def test_auto_dtype(self): @skip_pre_blackwell def test_nvfp4(self): + time.sleep(3600) model_path = f"{llm_models_root()}/nvfp4-quantized/Meta-Llama-3.1-8B" with LLM(model_path) as llm: assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 @@ -3636,6 +3638,7 @@ def test_fp8_block_scales(self, tp_size, pp_size, ep_size, attention_dp, ) def test_nvfp4(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, overlap_scheduler, moe_backend, eagle3): + time.sleep(3600) if moe_backend == "TRTLLM" and (get_sm_version() == 120 or get_sm_version() == 121): diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index ef3549dc6328..7ddbc413d8c7 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -746,6 +746,7 @@ def test_disaggregated_multi_gpu_with_mpirun_trt_backend( indirect=True) def test_disaggregated_cuda_graph(disaggregated_test_root, llm_venv, disaggregated_example_root, llama_model_root): + time.sleep(3600) src_dst_dict = { llama_model_root: f"{llm_venv.get_working_directory()}/TinyLlama/TinyLlama-1.1B-Chat-v1.0", diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index c1fe7c908958..c77963d56fd1 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -19,6 +19,7 @@ import subprocess import sys import tempfile +import time from pathlib import Path from typing import Any, Optional, Tuple, Union @@ -1678,6 +1679,7 @@ def test_openai_lora(llm_root, llm_venv): def test_openai_chat_multimodal_example(llm_root, llm_venv): + time.sleep(3600) test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd( ["-m", "pytest", diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 1b79caaec78e..d868a90d8523 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -31,7 +31,7 @@ l0_a10: - unittest/disaggregated/test_cluster_storage.py - disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_with_mpirun_trt_backend[TinyLlama-1.1B-Chat-v1.0] - - disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] + - disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] TIMEOUT (3) - disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] @@ -49,7 +49,7 @@ l0_a10: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0] - test_e2e.py::test_openai_chat_guided_decoding - - test_e2e.py::test_openai_chat_multimodal_example ISOLATION + - test_e2e.py::test_openai_chat_multimodal_example ISOLATION, TIMEOUT (3) - test_e2e.py::test_openai_perf_metrics - test_e2e.py::test_openai_prometheus - test_e2e.py::test_openai_lora diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index d1c39147b855..84d169518fbc 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -21,6 +21,6 @@ l0_b300: - unittest/_torch/thop/serial - unittest/_torch/executor # 250s - unittest/_torch/modules # 300s - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 + - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 TIMEOUT (3) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml index 57c3b6fd8106..1064dcdf5498 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml @@ -37,5 +37,5 @@ l0_gb200_multi_nodes: - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp] TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp] TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] TIMEOUT (90) - - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] TIMEOUT (90) + - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] TIMEOUT (3) - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp] TIMEOUT (90) From b27461e1f868c6c81bb0b02a73f29c5cf284c140 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 24 Nov 2025 08:34:40 +0000 Subject: [PATCH 03/10] fix pre-commit Signed-off-by: Yiqing Yan --- tests/integration/defs/utils/periodic_junit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/utils/periodic_junit.py b/tests/integration/defs/utils/periodic_junit.py index f7c49167a5e8..26ff6ba8e511 100644 --- a/tests/integration/defs/utils/periodic_junit.py +++ b/tests/integration/defs/utils/periodic_junit.py @@ -73,7 +73,8 @@ def __init__( interval: int = 18000, # Default 5 hours batch_size: int = 10, logger=None, # Optional logger (info, warning functions) - save_unfinished_test: bool = False, # Save unfinished test name in output-dir/unfinished_test.txt if True + save_unfinished_test: + bool = False, # Save unfinished test name in output-dir/unfinished_test.txt if True ): """ Initialize periodic reporter. From e7f82ef8b12267438878fd8eadf9d9b46eb4b35f Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 24 Nov 2025 09:33:02 +0000 Subject: [PATCH 04/10] test code Signed-off-by: Yiqing Yan --- tests/integration/defs/disaggregated/test_disaggregated.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 7ddbc413d8c7..b90bf93f1fef 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -18,6 +18,7 @@ import re import subprocess import tempfile +import time from typing import Callable import pytest From 392ab4a39ac2feaad7959ade59cdaec219bdf7ff Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 24 Nov 2025 11:00:52 +0000 Subject: [PATCH 05/10] revert test code Signed-off-by: Yiqing Yan --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 3 --- tests/integration/defs/disaggregated/test_disaggregated.py | 2 -- tests/integration/defs/test_e2e.py | 2 -- tests/integration/test_lists/test-db/l0_a10.yml | 4 ++-- tests/integration/test_lists/test-db/l0_b300.yml | 2 +- tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml | 2 +- 6 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 144e2abeffa4..fa82a863605a 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import time import pytest import torch @@ -52,7 +51,6 @@ def test_auto_dtype(self): @skip_pre_blackwell def test_nvfp4(self): - time.sleep(3600) model_path = f"{llm_models_root()}/nvfp4-quantized/Meta-Llama-3.1-8B" with LLM(model_path) as llm: assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 @@ -3638,7 +3636,6 @@ def test_fp8_block_scales(self, tp_size, pp_size, ep_size, attention_dp, ) def test_nvfp4(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, overlap_scheduler, moe_backend, eagle3): - time.sleep(3600) if moe_backend == "TRTLLM" and (get_sm_version() == 120 or get_sm_version() == 121): diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index b90bf93f1fef..ef3549dc6328 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -18,7 +18,6 @@ import re import subprocess import tempfile -import time from typing import Callable import pytest @@ -747,7 +746,6 @@ def test_disaggregated_multi_gpu_with_mpirun_trt_backend( indirect=True) def test_disaggregated_cuda_graph(disaggregated_test_root, llm_venv, disaggregated_example_root, llama_model_root): - time.sleep(3600) src_dst_dict = { llama_model_root: f"{llm_venv.get_working_directory()}/TinyLlama/TinyLlama-1.1B-Chat-v1.0", diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index c77963d56fd1..c1fe7c908958 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -19,7 +19,6 @@ import subprocess import sys import tempfile -import time from pathlib import Path from typing import Any, Optional, Tuple, Union @@ -1679,7 +1678,6 @@ def test_openai_lora(llm_root, llm_venv): def test_openai_chat_multimodal_example(llm_root, llm_venv): - time.sleep(3600) test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd( ["-m", "pytest", diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index d868a90d8523..1b79caaec78e 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -31,7 +31,7 @@ l0_a10: - unittest/disaggregated/test_cluster_storage.py - disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_single_gpu_with_mpirun_trt_backend[TinyLlama-1.1B-Chat-v1.0] - - disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] TIMEOUT (3) + - disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] @@ -49,7 +49,7 @@ l0_a10: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0] - test_e2e.py::test_openai_chat_guided_decoding - - test_e2e.py::test_openai_chat_multimodal_example ISOLATION, TIMEOUT (3) + - test_e2e.py::test_openai_chat_multimodal_example ISOLATION - test_e2e.py::test_openai_perf_metrics - test_e2e.py::test_openai_prometheus - test_e2e.py::test_openai_lora diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 84d169518fbc..d1c39147b855 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -21,6 +21,6 @@ l0_b300: - unittest/_torch/thop/serial - unittest/_torch/executor # 250s - unittest/_torch/modules # 300s - - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 TIMEOUT (3) + - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml index 1064dcdf5498..57c3b6fd8106 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml @@ -37,5 +37,5 @@ l0_gb200_multi_nodes: - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp] TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp] TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] TIMEOUT (90) - - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] TIMEOUT (3) + - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp] TIMEOUT (90) From 054a0c8f2caf78ebfedd6c00c6f2f5f769023076 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Wed, 26 Nov 2025 09:06:10 +0000 Subject: [PATCH 06/10] remove --periodic-interval since --periodic-batch-size=1 Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 94e612c7f571..f5f0207a2c85 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -798,7 +798,6 @@ def getPytestBaseCommandLine( "--cov-report=", "--cov-config=${coverageConfigFile}", "--periodic-junit", - "--periodic-interval=1800", // 30 minutes "--periodic-batch-size=1", "--periodic-save-unfinished-test", ] From 50d7538e2ed84429a95c8b3834e3ae36133836b9 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Thu, 27 Nov 2025 07:31:52 +0000 Subject: [PATCH 07/10] fix periodic junit xml file path Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index f5f0207a2c85..fd571c4c2cb4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -181,12 +181,12 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { !cmd.contains("--test-list=") && !cmd.contains("--test-prefix=") && !cmd.contains("--csv=") && - !cmd.contains("--junit-xml") + !cmd.contains("--periodic-junit-xmlpath") } isolateTestCmdLine += ["--test-list=${singleTestFile}"] isolateTestCmdLine += ["--test-prefix=${stageName}"] isolateTestCmdLine += ["--csv=${WORKSPACE}/${stageName}/report_isolated_${i}.csv"] - isolateTestCmdLine += ["--junit-xml ${WORKSPACE}/${stageName}/results_isolated_${i}.xml"] + isolateTestCmdLine += ["--periodic-junit-xmlpath ${WORKSPACE}/${stageName}/results_isolated_${i}.xml"] isolateTestCmdLine += ["--cov-append"] // Append coverage data to avoid overwriting previous data try { @@ -790,7 +790,6 @@ def getPytestBaseCommandLine( "--waives-file=${waivesFilePath}", "--output-dir=${outputPath}/", "--csv=${outputPath}/report.csv", - "--junit-xml ${outputPath}/results.xml", "-o junit_logging=out-err", "--cov=${llmSrc}/examples/", "--cov=${llmSrc}/tensorrt_llm/", @@ -798,6 +797,7 @@ def getPytestBaseCommandLine( "--cov-report=", "--cov-config=${coverageConfigFile}", "--periodic-junit", + "--periodic-junit-xmlpath ${outputPath}/results.xml", "--periodic-batch-size=1", "--periodic-save-unfinished-test", ] @@ -1983,14 +1983,14 @@ def rerunFailedTests(stageName, llmSrc, testCmdLine, resultFileName="results.xml def xmlFile = "${rerunDir}/rerun_results_${times}.xml" // change the testCmdLine for rerun def noNeedLine = ["--splitting-algorithm", "--splits", "--group", "--cov"] - def needToChangeLine = ["--test-list", "--csv", "--junit-xml"] + def needToChangeLine = ["--test-list", "--csv", "--periodic-junit-xmlpath"] def newTestCmdLine = testCmdLine.findAll { cmd -> !noNeedLine.any { line -> cmd.contains(line) } && !needToChangeLine.any { line -> cmd.contains(line) } } newTestCmdLine += [ "--test-list=${currentRerunTestList}", "--csv=${rerunDir}/rerun_report_${times}.csv", - "--junit-xml ${xmlFile}", + "--periodic-junit-xmlpath ${xmlFile}", "--reruns ${times - 1}" ] try { From 988414a93a848e0abe1805916bbf8e6a62e38d8d Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Fri, 28 Nov 2025 06:54:51 +0000 Subject: [PATCH 08/10] fix based on review comments Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index fd571c4c2cb4..a681a31645c7 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -129,13 +129,12 @@ def uploadResults(def pipeline, SlurmCluster cluster, String nodeName, String st sh "mkdir -p ${stageName}" // Download timeout test results def timeoutTestFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/unfinished_test.txt" - def downloadTimeoutTestCmd = "sshpass -p '${remote.passwd}' scp -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${timeoutTestFilePath} ${stageName}/" - def downloadTimeoutTestSucceed = sh(script: downloadTimeoutTestCmd, returnStatus: true) == 0 + def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -P ${remote.port} -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${timeoutTestFilePath} ${stageName}/", returnStatus: true, numRetries: 3) == 0 if (downloadTimeoutTestSucceed) { sh "ls ${stageName}" def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") if (timeoutTestXml != null) { - sh "echo '${timeoutTestXml}' > ${stageName}/results-timeout.xml" + writeFile file: "${stageName}/results-timeout.xml", text: timeoutTestXml hasTimeoutTest = true } } @@ -1279,7 +1278,7 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu if (stageIsFailed) { def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") if (timeoutTestXml != null) { - sh "echo '${timeoutTestXml}' > ${stageName}/results-timeout.xml" + writeFile file: "${stageName}/results-timeout.xml", text: timeoutTestXml } def stageXml = generateStageFailTestResultXml(stageName, "Stage Failed", "Stage run failed without result", "results*.xml") if (stageXml != null) { @@ -1642,6 +1641,10 @@ def launchTestListCheck(pipeline) } def generateTimeoutTestResultXml(stageName, testFilePath) { + if (!fileExists("${stageName}/${testFilePath}")) { + echo "No ${testFilePath} found in ${stageName}, skipping timeout XML generation" + return null + } String timeoutTests = sh(script: "cd ${stageName} && cat ${testFilePath}", returnStdout: true).trim() echo "timeoutTests: ${timeoutTests}" From f503d806cd123ea71b430694dadcfeffea921b01 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Mon, 1 Dec 2025 08:19:15 +0000 Subject: [PATCH 09/10] update timeout test junit xml Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a681a31645c7..ed59fdd08901 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1656,8 +1656,8 @@ def generateTimeoutTestResultXml(stageName, testFilePath) { """ testList.each { test -> xmlContent += """ - Test timeout - """ + Test terminated unexpectedly + """ } xmlContent += "" return xmlContent From 0eccfee5e214b1969094e04fd11472bacf0e92d4 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Tue, 2 Dec 2025 10:43:00 +0000 Subject: [PATCH 10/10] update timeout junit xml Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ed59fdd08901..027dd6767d2e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -134,7 +134,11 @@ def uploadResults(def pipeline, SlurmCluster cluster, String nodeName, String st sh "ls ${stageName}" def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") if (timeoutTestXml != null) { - writeFile file: "${stageName}/results-timeout.xml", text: timeoutTestXml + sh """ +cat > ${stageName}/results-timeout.xml << 'EOF_TIMEOUT_XML' +${timeoutTestXml} +EOF_TIMEOUT_XML + """ hasTimeoutTest = true } } @@ -1278,7 +1282,11 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu if (stageIsFailed) { def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt") if (timeoutTestXml != null) { - writeFile file: "${stageName}/results-timeout.xml", text: timeoutTestXml + sh """ +cat > ${stageName}/results-timeout.xml << 'EOF_TIMEOUT_XML' +${timeoutTestXml} +EOF_TIMEOUT_XML + """ } def stageXml = generateStageFailTestResultXml(stageName, "Stage Failed", "Stage run failed without result", "results*.xml") if (stageXml != null) {