From 290cdb27e81dd00a947c27cf8cf5477f24521977 Mon Sep 17 00:00:00 2001 From: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:21:23 -0700 Subject: [PATCH 1/3] [TRTLLM-14155][perf] Reland host-side greedy stop checks Reland the host-side greedy stop-criteria fast path from #15920 after its temporary revert in #16163. Signed-off-by: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> --- .../_torch/pyexecutor/sampler/sampler.py | 121 +++++++++++++----- .../_torch/sampler/test_torch_sampler.py | 101 +++++++++++++++ 2 files changed, 188 insertions(+), 34 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index 1ad625d6d234..6726a430ef97 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -1113,6 +1113,8 @@ def finish_reasons_list(self) -> FinishReasonsList: @dataclass(kw_only=True) class SampleStateTorch(SampleState[SampleStateTensorsHostTorch, SampleStateTensors]): beam_history_builders: list[BeamHistoryBuilder | None] | None = None + use_host_stop_criteria: bool = False + """Whether update_requests should evaluate end-ID and length limits on the host.""" @dataclass(kw_only=True, frozen=True) @@ -2452,6 +2454,21 @@ def _can_use_fast_greedy_path(self, requests: list[LlmRequest]) -> bool: return False return True + def _can_use_host_stop_criteria(self, requests: list[LlmRequest]) -> bool: + """Check whether stop criteria can be evaluated from the host token copy. + + The non-speculative, single-beam path already copies sampled tokens to + the host. When no request has stop words, checking end IDs and length + limits on the host avoids the device finish-reason kernels and their + additional D2H copy. + """ + return ( + bool(requests) + and self.max_tokens == 1 + and self.max_beam_width == 1 + and all(not req.py_is_draft and not req.py_stop_words_list for req in requests) + ) + @staticmethod def _meet_max_token_stop_criteria( request: LlmRequest, max_seq_len: int, beam_idx: int = DEFAULT_BEAM_IDX @@ -3719,13 +3736,23 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: req.py_rewind_len = 0 else: processed = 1 - num_accepted = self.process_draft_tokens( - req, - new_tokens_tensor=new_tokens, - new_tokens_list=new_tokens_list, - finish_reasons=finish_reasons, - resource_manager=resource_manager, - ) + if state.use_host_stop_criteria: + new_token = add_token(req, new_tokens_list, beam_idx=DEFAULT_BEAM_IDX) + self._handle_stop_criteria( + req, + new_token, + max_seq_len=self.max_seq_len, + beam_idx=DEFAULT_BEAM_IDX, + ) + num_accepted = 0 + else: + num_accepted = self.process_draft_tokens( + req, + new_tokens_tensor=new_tokens, + new_tokens_list=new_tokens_list, + finish_reasons=finish_reasons, + resource_manager=resource_manager, + ) if (actual_draft_len := get_draft_token_length(req)) > 0: req.py_num_accepted_draft_tokens = num_accepted req.py_rewind_len = actual_draft_len - num_accepted @@ -3784,10 +3811,10 @@ def sample_async( ( requests, seq_slots_host, - seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, + use_host_stop_criteria, ) = self._process_requests( scheduled_requests, model_outputs, @@ -3801,7 +3828,8 @@ def sample_async( # Forwarded to _record_sampler_event so SamplerEvent.synchronize # awaits any side-stream D2H copies host-side. side_stream_event: torch.cuda.Event | None = None - if requests: + if requests and not use_host_stop_criteria: + assert seq_lens_cuda is not None beam_search_store = self.store.beam_search_store assert self._use_beam_search == (beam_search_store is not None) # Prepare stop word handling @@ -3867,6 +3895,7 @@ def sample_async( ), sampler_event=sampler_event, beam_history_builders=beam_history_builders, + use_host_stop_criteria=use_host_stop_criteria, ) @staticmethod @@ -4650,7 +4679,12 @@ def _process_requests( new_tokens_cuda: torch.Tensor, num_context_logits_prefix_sum: list[int], ) -> tuple[ - list[LlmRequest], torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor + list[LlmRequest], + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + torch.Tensor, + bool, ]: raw_logits_cuda = model_outputs["logits"] @@ -4663,6 +4697,11 @@ def _process_requests( if return_log_probs: self._prepare_log_probs(sampling_requests) + use_fast_greedy_path = self._can_use_fast_greedy_path(sampling_requests) + use_host_stop_criteria = use_fast_greedy_path and self._can_use_host_stop_criteria( + sampling_requests + ) + seq_slots_host = torch.tensor( [r.py_seq_slot for r in sampling_requests], dtype=torch.int32, @@ -4670,18 +4709,24 @@ def _process_requests( ) # necessary for beam search and max_length checks - seq_lens_host = torch.tensor( - [r.max_beam_num_tokens for r in sampling_requests], - dtype=torch.int32, - pin_memory=prefer_pinned(), + seq_lens_host = ( + None + if use_host_stop_criteria + else torch.tensor( + [r.max_beam_num_tokens for r in sampling_requests], + dtype=torch.int32, + pin_memory=prefer_pinned(), + ) ) - # Cast seq_slots / seq_lens to CUDA exactly once; consumed by both - # the per-group beam-search metadata builder and the finish-reasons - # handler in sample_async. int64 is required for the index_*_ ops - # downstream. + # Cast seq_slots / seq_lens to CUDA exactly once. The fast host stop- + # criteria path only needs seq_slots for scattering sampled tokens; + # all other paths also consume seq_lens in device-side finish handling + # or beam-search metadata. seq_slots_cuda = seq_slots_host.to(device="cuda", dtype=torch.int64, non_blocking=True) - seq_lens_cuda = seq_lens_host.to(device="cuda", non_blocking=True) + seq_lens_cuda = ( + None if seq_lens_host is None else seq_lens_host.to(device="cuda", non_blocking=True) + ) # Handle embedding bias self._apply_embedding_bias( @@ -4696,19 +4741,25 @@ def _process_requests( ) # Fast path for greedy sampling - if self._can_use_fast_greedy_path(sampling_requests): - # Compute destination indices on CPU (same pattern as _unbatch_sampling_results) - batch_destination_indexer = _UnpackedStepIndexer( - seq_slots=seq_slots_host, - num_steps=sampling_requests_metadata.req_num_generated_tokens, - steps_dim_size=new_tokens_cuda.size(0), - slots_dim_size=new_tokens_cuda.size(1), - dim_order=_UnpackedStepIndexer.DimOrder.STEP_MAJOR, - index_dtype=torch.int64, - ) - batch_dest_indices_cuda = batch_destination_indexer[:].to( - new_tokens_cuda.device, non_blocking=True - ) + if use_fast_greedy_path: + if use_host_stop_criteria: + # There is exactly one token and one beam per request, so the + # linearized destination indices are the sequence slots. + batch_dest_indices_cuda = seq_slots_cuda + else: + # Compute destination indices on CPU (same pattern as + # _unbatch_sampling_results). + batch_destination_indexer = _UnpackedStepIndexer( + seq_slots=seq_slots_host, + num_steps=sampling_requests_metadata.req_num_generated_tokens, + steps_dim_size=new_tokens_cuda.size(0), + slots_dim_size=new_tokens_cuda.size(1), + dim_order=_UnpackedStepIndexer.DimOrder.STEP_MAJOR, + index_dtype=torch.int64, + ) + batch_dest_indices_cuda = batch_destination_indexer[:].to( + new_tokens_cuda.device, non_blocking=True + ) # Get d2t tensor if present d2t = model_outputs.get("d2t", None) @@ -4726,10 +4777,10 @@ def _process_requests( return ( sampling_requests, seq_slots_host, - seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, + use_host_stop_criteria, ) # Indexer for accessing tokens in 'logits_cuda', corresponding to the @@ -4742,6 +4793,8 @@ def _process_requests( ) # Perform sampling in batches + assert seq_lens_host is not None + assert seq_lens_cuda is not None batched_sampling_result = self._sample_batched_by_strategy( logits_cuda, sampling_requests, @@ -4779,10 +4832,10 @@ def _process_requests( return ( sampling_requests, seq_slots_host, - seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, + use_host_stop_criteria, ) @override diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index 87e6b4b8dc01..b76d4aba0d16 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -866,6 +866,107 @@ def test_write_finish_reasons(cls): run_test_with_warmup(uut_provider, max_sync_s=0.5) + @pytest.mark.parametrize( + "sampled_token,end_id,max_new_tokens,expected_reason", + [ + pytest.param(42, 99, 5, None, id="not-finished"), + pytest.param(99, 99, 5, FinishReason.END_ID, id="end-id"), + pytest.param(42, 99, 1, FinishReason.LENGTH, id="length"), + pytest.param( + 99, + 99, + 1, + FinishReason.END_ID, + id="end-id-precedes-length", + ), + ], + ) + def test_host_stop_criteria_fast_path( + self, + mocker, + sampled_token: int, + end_id: int, + max_new_tokens: int, + expected_reason: FinishReason | None, + ): + sampler = TorchSampler( + TorchSampler.Args( + max_seq_len=20, + max_draft_len=0, + max_total_draft_tokens=0, + max_num_sequences=1, + max_beam_width=1, + ) + ) + request = LlmRequest( + request_id=0, + seq_slot=0, + input_tokens=[1, 2], + max_new_tokens=max_new_tokens, + end_id=end_id, + sampling_config=SamplingConfig(), + is_streaming=False, + ) + + setup_requests = ScheduledRequests() + setup_requests.context_requests_last_chunk = [request] + sampler.setup_sampler_step(setup_requests) + + scheduled_requests = ScheduledRequests() + scheduled_requests.generation_requests = [request] + logits = torch.full((1, 128), -1.0, dtype=torch.float32, device="cuda") + logits[0, sampled_token] = 1.0 + + finish_by = mocker.spy(request, "finish_by") + write_finish_reasons = mocker.patch.object( + sampler._finish_reasons_handler, + "write_finish_reasons", + side_effect=AssertionError("device finish reasons should be skipped"), + ) + + state = sampler.sample_async( + scheduled_requests, + model_outputs={"logits": logits}, + num_context_logits_prefix_sum=[0], + ) + assert state.use_host_stop_criteria + assert state.host is not None + assert state.host.finish_reasons is None + + sampler.update_requests(state) + + write_finish_reasons.assert_not_called() + assert request.get_tokens(0)[-1] == sampled_token + if expected_reason is None: + finish_by.assert_not_called() + else: + finish_by.assert_called_once_with(expected_reason, 0) + + @pytest.mark.parametrize( + "sample_request", + [ + pytest.param( + SimpleNamespace(py_is_draft=False, py_stop_words_list=[[42], [1]]), + id="stop-words", + ), + pytest.param( + SimpleNamespace(py_is_draft=True, py_stop_words_list=None), + id="draft-request", + ), + ], + ) + def test_host_stop_criteria_fast_path_fallback(self, sample_request): + sampler = TorchSampler( + TorchSampler.Args( + max_seq_len=20, + max_draft_len=0, + max_total_draft_tokens=0, + max_num_sequences=1, + max_beam_width=1, + ) + ) + assert not sampler._can_use_host_stop_criteria([cast(LlmRequest, sample_request)]) + @classmethod def test_are_stop_words_isnt_called_when_no_stop_words(cls, monkeypatch: pytest.MonkeyPatch): """We don't want to call are_stop_words when there are no stop words because it's expensive""" From c0c45c7b2c29a58e8b4b29f22a83ef760fbcd9b9 Mon Sep 17 00:00:00 2001 From: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:03:52 -0700 Subject: [PATCH 2/3] [None][fix] Propagate host stop criteria across PP ranks Signed-off-by: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/sampler/sampler.py | 8 ++++---- tests/unittest/_torch/sampler/test_torch_sampler.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index 6726a430ef97..e92c40886820 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -1099,6 +1099,8 @@ class SampleStateTensorsHostTorch(SampleStateTensors): finish_reasons: torch.Tensor | None first_finish_reasons: torch.Tensor | None logprobs_state: LogProbsState | None = None + use_host_stop_criteria: bool = False + """Whether update_requests should evaluate end-ID and length limits on the host.""" def finish_reasons_list(self) -> FinishReasonsList: """`(num_seq_slots, num_steps)`""" @@ -1113,8 +1115,6 @@ def finish_reasons_list(self) -> FinishReasonsList: @dataclass(kw_only=True) class SampleStateTorch(SampleState[SampleStateTensorsHostTorch, SampleStateTensors]): beam_history_builders: list[BeamHistoryBuilder | None] | None = None - use_host_stop_criteria: bool = False - """Whether update_requests should evaluate end-ID and length limits on the host.""" @dataclass(kw_only=True, frozen=True) @@ -3736,7 +3736,7 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: req.py_rewind_len = 0 else: processed = 1 - if state.use_host_stop_criteria: + if state.host.use_host_stop_criteria: new_token = add_token(req, new_tokens_list, beam_idx=DEFAULT_BEAM_IDX) self._handle_stop_criteria( req, @@ -3892,10 +3892,10 @@ def sample_async( finish_reasons=finish_reasons_host, first_finish_reasons=first_finish_reasons_host, logprobs_state=logprobs_state, + use_host_stop_criteria=use_host_stop_criteria, ), sampler_event=sampler_event, beam_history_builders=beam_history_builders, - use_host_stop_criteria=use_host_stop_criteria, ) @staticmethod diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index b76d4aba0d16..4da8ab9dd83e 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -42,6 +42,7 @@ get_draft_token_length, ) from tensorrt_llm._torch.pyexecutor.sampler import ( + SampleStateTorch, TorchSampler, _BatchedSamplingResult, _request_get_sampling_params, @@ -929,11 +930,18 @@ def test_host_stop_criteria_fast_path( model_outputs={"logits": logits}, num_context_logits_prefix_sum=[0], ) - assert state.use_host_stop_criteria assert state.host is not None + assert state.host.use_host_stop_criteria assert state.host.finish_reasons is None - sampler.update_requests(state) + # PP ranks only receive the host state, so all state required by + # update_requests must survive reconstructing the outer sample state. + transferred_state = SampleStateTorch( + requests=state.requests, + host=state.host, + sampler_event=state.sampler_event, + ) + sampler.update_requests(transferred_state) write_finish_reasons.assert_not_called() assert request.get_tokens(0)[-1] == sampled_token From 009540215703bc43b7f98eba0e87d466915b121f Mon Sep 17 00:00:00 2001 From: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:41:17 -0700 Subject: [PATCH 3/3] [https://nvbugs/6427411][test] Re-enable PP regression tests Remove the PP sampler waivers introduced by #16103, #16105, #16127, and #16146. Also remove overlapping PP and gpu2 waivers from #16109, plus the PP2 waiver added by #16169, so the restored pre-#15920 sampler path is exercised in CI. Signed-off-by: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 51 ------------------------- 1 file changed, 51 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 97a424baa847..ea92de5dbfc2 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -1,24 +1,10 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6396413) accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6396415) -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp2tp1cp2] SKIP (https://nvbugs/6427411) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] SKIP (https://nvbugs/6426865) accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6075533) accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] SKIP (https://nvbugs/6418103) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4] SKIP (https://nvbugs/6428069) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram SKIP (https://nvbugs/6245651) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] SKIP (https://nvbugs/6427411) accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6402054) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] SKIP (https://nvbugs/6336747) @@ -54,31 +40,19 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2- accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6426847) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6402058) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428057) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6388153) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6388153) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6428094) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428096) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6198785) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198785) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP] SKIP (https://nvbugs/6313993) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6388153) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6388363) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6428087) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5945081) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6384625) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6428063) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6384625) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6384625) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6272673) @@ -103,17 +77,10 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_ba accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True] SKIP (https://nvbugs/6422337) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/5616182) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6278337) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6428089) -accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] SKIP (https://nvbugs/6255417) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] SKIP (https://nvbugs/6177390) @@ -133,7 +100,6 @@ accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_ accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] SKIP (https://nvbugs/6336747) accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4] SKIP (https://nvbugs/6336747) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] SKIP (https://nvbugs/6367805) -accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6427411) cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689) cpp/test_e2e.py::test_benchmarks[gpt-80] SKIP (https://nvbugs/5550689) cpp/test_e2e.py::test_model[-bart-90] SKIP (https://nvbugs/6162804) @@ -147,11 +113,6 @@ cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (https://nvbugs/5838199) cpp/test_unit_tests.py::test_unit_tests[executor-80] SKIP (https://nvbugs/6412076) disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6427411) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6427411) -disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6428069) -disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6427411) -disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6428069) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_cache_aware_balance[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_conditional[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_gen_only[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) @@ -190,8 +151,6 @@ examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (https examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) -examples/test_ray.py::test_llm_inference_distributed_ray[pp2] SKIP (https://nvbugs/6427411) -examples/test_ray.py::test_llm_inference_distributed_ray[tp2pp2] SKIP (https://nvbugs/6427411) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502) examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2i_lpips_against_golden SKIP (https://nvbugs/6418815) @@ -401,7 +360,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6323074) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6379406) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418834) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6428144) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418834) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6422339) @@ -425,7 +383,6 @@ triton_server/test_triton.py::test_python_bls_unit_tests[python-bls-unit-tests] unittest/_torch/attention/sparse/deepseek_v4/test_compressor_kernel.py::test_prefill_varlen[varlen_hd512_overlap] SKIP (https://nvbugs/6426860) unittest/_torch/misc/test_share_tensor.py::TestShareTensor::test_share_tensor_different_dtypes SKIP (https://nvbugs/6418021) unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) -unittest/_torch/modules/tests_lora_modules/test_nemotron_h_lora_sanity.py::TestNemotronHLoRA::test_lora_pp2_sanity SKIP (https://nvbugs/6428124) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part0" SKIP (https://nvbugs/6372711) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part1" SKIP (https://nvbugs/6426852) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_partial_update_weights_nvfp4[auto-Qwen3/Qwen3-8B] SKIP (https://nvbugs/6372690) @@ -455,15 +412,7 @@ unittest/disaggregated/test_kv_transfer.py::test_transfer_worker_v2[tp4_pp1_to_t unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) -unittest/llmapi/test_additional_model_outputs.py -m "gpu2" SKIP (https://nvbugs/6428091) -unittest/llmapi/test_additional_model_outputs.py::test_additional_model_outputs_integration_pp2 SKIP (https://nvbugs/6427411) -unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) -unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) -unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-False-True] SKIP (https://nvbugs/6427411) -unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-True-True] SKIP (https://nvbugs/6427411) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_phi3_lora_fused_modules_output_on_tp2_identical_to_tp1 SKIP (https://nvbugs/6109745) -unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_2gpu[1-2] SKIP (https://nvbugs/6427411) -unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6427411) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP (https://nvbugs/6162504) unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781)