From ac9361839705b687ef7d1f27cd8b15bcac318c42 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Fri, 15 May 2026 19:05:46 -0700 Subject: [PATCH] [nvbugs/6018046][fix] Tune throughput_pp4_mtp budget to match working bs8_mtp The throughput_pp4_mtp variant uses pp_size=4 with ep_size=1, which keeps all 256 experts on every rank rather than sharding them. Per-GPU weight footprint is therefore much higher than the EP-sharded throughput_mtp/throughput_tp8 variants, leaving little headroom on top of a 0.70 KV-cache fraction. Under the post-merge B300 load, the runtime cuBLAS / NCCL workspaces would intermittently hit allocator pressure and the router GEMM in DeepseekV3Gate faulted with CUBLAS_STATUS_EXECUTION_FAILED (followed by an asynchronous illegal-memory-access). The Repair Bot already showed the test passes deterministically on a fresh B300 node, so the fix is to bring this variant in line with the already-working throughput_bs8_mtp configuration: * drop max_batch_size from 32 -> 8 (same as throughput_bs8_mtp), shrinking activation / CUDA-graph / NCCL workspace memory * lower the KV cache fraction to 0.5 specifically for the pp_size>1 + ep_size=1 + mtp_nextn>0 path so prefill workspaces fit Removes the matching waiver entry. Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- .../integration/defs/accuracy/test_llm_api_pytorch.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 8b4d2c499860..49bb7dcd7929 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2738,7 +2738,7 @@ class TestDeepSeekR1(LlmapiAccuracyTestHarness): False, True, True, - 32, + 8, "CUTLASS", marks=pytest.mark.skip_less_mpi_world_size(4)), ], @@ -2756,7 +2756,14 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, if moe_backend == "TRTLLM" and sm_version in (120, 121): pytest.skip(f"{moe_backend} backend does not support SM 120 or 121") - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.70) + # PP4 + MTP + ep_size=1 keeps all 256 experts on every rank, so the + # weight footprint per GPU is much higher than the EP-sharded variants. + # The default 0.70 KV fraction leaves too little headroom for runtime + # NCCL / cuBLAS workspaces on Blackwell, which manifests as mid-run + # CUBLAS_STATUS_EXECUTION_FAILED in the router GEMM (NVBug 6018046). + kv_fraction = 0.5 if (pp_size > 1 and ep_size == 1 + and mtp_nextn > 0) else 0.70 + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=kv_fraction) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, cuda_graph_config=CudaGraphConfig() if cuda_graph else None,