From a6ba7676ecfc42ef0e29a64d271ca78df4e5e077 Mon Sep 17 00:00:00 2001 From: qqiao Date: Wed, 14 May 2025 18:35:46 -0700 Subject: [PATCH 1/5] Try another timeout opt to kill test thread Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 8fc738e598ed..444064559552 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1013,6 +1013,8 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO extraInternalEnv, "pytest", "-v", + "-s", + "--timeout-method=thread", "--apply-test-list-correction", "--splitting-algorithm least_duration", "--timeout=${pytestTestTimeout}", From a50bc0f3879efb04a06d932f5a38fe1496c7df21 Mon Sep 17 00:00:00 2001 From: qqiao Date: Wed, 14 May 2025 23:46:05 -0700 Subject: [PATCH 2/5] Return true when try to delete non-existing result file Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 444064559552..2258bd822fc4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1013,7 +1013,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO extraInternalEnv, "pytest", "-v", - "-s", "--timeout-method=thread", "--apply-test-list-correction", "--splitting-algorithm least_duration", @@ -1124,7 +1123,7 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG sh """ ls -all ${stageName}/ if ! grep -q ' Date: Wed, 14 May 2025 23:50:59 -0700 Subject: [PATCH 3/5] quick test for the result file Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 2258bd822fc4..9c96dd88e787 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -996,7 +996,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def extraInternalEnv = "" // Move back to 3600 once TRTLLM-4000 gets resolved - def pytestTestTimeout = "7200" + def pytestTestTimeout = "600" // TRT uses half of the host logic cores for engine building which is bad for multi-GPU machines. extraInternalEnv = "__LUNOWUD=\"-thread_pool_size=${TESTER_CORES}\"" From 58bdd039bb60396a251563a682dd439761685e54 Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 15 May 2025 03:55:07 -0700 Subject: [PATCH 4/5] Change back the global timeout setting Signed-off-by: qqiao --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9c96dd88e787..2258bd822fc4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -996,7 +996,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def extraInternalEnv = "" // Move back to 3600 once TRTLLM-4000 gets resolved - def pytestTestTimeout = "600" + def pytestTestTimeout = "7200" // TRT uses half of the host logic cores for engine building which is bad for multi-GPU machines. extraInternalEnv = "__LUNOWUD=\"-thread_pool_size=${TESTER_CORES}\"" From cb2bf5ebc1d5909fa03f3204e7ffb6d39afc7ec7 Mon Sep 17 00:00:00 2001 From: qqiao Date: Thu, 15 May 2025 03:59:52 -0700 Subject: [PATCH 5/5] Try to kill test in internal pytest Signed-off-by: qqiao --- tests/integration/defs/test_unittests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index b38a8a9fa3c4..8f8cd2f987e2 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -122,6 +122,7 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir): ignore_opt, "-v", "--timeout=1600", + "--timeout-method=thread", ] if dry_run: