From 09dfa5912fcae685b06476d487016618f5fef6c0 Mon Sep 17 00:00:00 2001 From: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> Date: Mon, 11 May 2026 21:54:09 -0700 Subject: [PATCH 1/2] [None][fix] Raise server_waiting_timeout to 3600s for DSv4 disagg tests V4-Flash safetensors prefetch (~148 GB at ~90 MB/s on shared scratch) plus autotuner and CUDA graph warmup needs ~37 min/worker at TP=2, exceeding the 2100s (35 min) default. The wait loop times out, then pytest.fail short-circuits the with-block before yield, and cleanup hangs in Popen.wait() until the 60 min @pytest.mark.timeout kills it. Signed-off-by: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> --- .../defs/accuracy/test_disaggregated_serving.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 505683dcc222..6272fad9494e 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2269,9 +2269,12 @@ def test_auto_dtype(self): "num_instances": 1 }, } + # V4-Flash 148GB weight prefetch + warmup needs >35 min, default wait timeout times out. with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: + ctx_server_config, + gen_server_config, + self.MODEL_PATH, + server_waiting_timeout=3600) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm, is_integration_test=True) @@ -2330,8 +2333,11 @@ def test_auto_dtype(self): "num_instances": 1 }, } + # Same long-init reason as TestDeepSeekV4Flash above. with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: + ctx_server_config, + gen_server_config, + self.MODEL_PATH, + server_waiting_timeout=3600) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm, is_integration_test=True) From 84107837d421717e31bc4e00b03abe0ee4e2401e Mon Sep 17 00:00:00 2001 From: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> Date: Tue, 12 May 2026 00:46:42 -0700 Subject: [PATCH 2/2] Revert "[None][chore] waive TestDeepSeekV4Flash::test_auto_dtype (#14024)" This reverts commit f2af68f82d4510a1c2f1f767535914965baf4f5c. Signed-off-by: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f53af6b823f1..e38aa988a437 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -430,4 +430,3 @@ disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-cutlass-auto] SKIP (https://nvbugs/5596343) unittest/_torch/speculative/test_eagle3.py::test_llama_eagle3_dynamic_tree[True-False] SKIP (https://nvbugs/6113021) accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] SKIP (worker init CUDA error masked by kv_cache_manager_v2/_exceptions.py:49 cuGetErrorString type bug; real failure unreadable until that wrapper is fixed) -accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype SKIP (https://nvbugs/6166283)