diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index dcf09c4567da..13015129d30e 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -187,6 +187,15 @@ def _prepare_qwen_vl_mrope_config( _MAX_PIXELS_TOKEN_PROBE = 1 << 31 +def _get_mrope_position_delta_cache_size( + model_config: ModelConfig[PretrainedConfig]) -> int: + """Return real sequence-slot capacity plus one reserved dummy slot.""" + max_num_seq_slots = model_config.extra_attrs.get( + 'max_num_seq_slots', + model_config.max_num_tokens * model_config.mapping.pp_size) + return max_num_seq_slots + 1 + + class Qwen2VLInputProcessorBase(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): @@ -1723,8 +1732,8 @@ def __init__( if not disable_fuse_rope: self.init_mrope_embedding(model_config) # Extra slot is reserved for CUDA graph / warmup dummy requests. - max_mrope_delta_slots = ( - model_config.max_num_tokens * model_config.mapping.pp_size + 1) + max_mrope_delta_slots = _get_mrope_position_delta_cache_size( + model_config) self.register_buffer('mrope_position_deltas_cache', torch.zeros(max_mrope_delta_slots, dtype=torch.int32, diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index a5acbfe509ff..acd8b57f733a 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -58,6 +58,7 @@ from .modeling_qwen2vl import ( Qwen2_5_VLVisionAttention, Qwen2VLInputProcessorBase, + _get_mrope_position_delta_cache_size, _prepare_qwen_vl_mrope_config, _prepare_qwen_vl_vision_attn_metadata, ) @@ -1216,7 +1217,7 @@ def __init__( if not disable_fuse_rope: self.init_mrope_embedding(model_config) # Extra slot is reserved for CUDA graph / warmup dummy requests. - max_mrope_delta_slots = model_config.max_num_tokens * model_config.mapping.pp_size + 1 + max_mrope_delta_slots = _get_mrope_position_delta_cache_size(model_config) self.register_buffer( "mrope_position_deltas_cache", torch.zeros( diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 9c0023cac331..a1e1caad4692 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -2333,41 +2333,24 @@ def create_kv_cache_compression_manager( return None -def compute_max_num_sequences(mapping: Mapping, - max_batch_size: int, - disable_overlap_scheduler: bool, - enable_overlap_headroom: bool = False) -> int: +def compute_max_num_sequences(mapping: Mapping, max_batch_size: int, + disable_overlap_scheduler: bool) -> int: """Size the sequence-slot pool (and the sampler state it indexes). - ``enable_overlap_headroom`` is intentionally opt-in. DeepSeek-V4 needs a - second non-PP slot set because the V2 scheduler can backfill seats before - the overlap scheduler releases the previous iteration's terminal slots. - Other models retain their established sizing until that behavior is - validated independently. Pipeline parallelism already sizes the pool by - ``pp_size``. + The overlap scheduler needs a second non-PP slot set because it can + backfill seats before releasing the previous iteration's terminal slots. + Pipeline parallelism already sizes the pool by ``pp_size``. """ if mapping.has_pp(): num_micro_batches = mapping.pp_size else: - num_micro_batches = (2 if enable_overlap_headroom - and not disable_overlap_scheduler else 1) + num_micro_batches = 1 if disable_overlap_scheduler else 2 return max_batch_size * num_micro_batches -def should_enable_dsv4_adp_dummy_fixes(model_type: Optional[str], - mapping: Mapping) -> bool: - """Gate DSv4 ADP dummy behavior while PP remains follow-up scope.""" - return model_type == "deepseek_v4" and not mapping.has_pp() - - -def should_enable_dsv4_overlap_headroom( - model_type: Optional[str], spec_config: Optional[SpeculativeConfig], - mapping: Mapping, disable_overlap_scheduler: bool) -> bool: - """Gate extra sequence slots to the validated DSv4 MTP overlap path.""" - return (should_enable_dsv4_adp_dummy_fixes(model_type, mapping) - and spec_config is not None - and spec_config.spec_dec_mode.is_mtp_eagle_one_model() - and not disable_overlap_scheduler) +def should_enable_adp_dummy_fixes(mapping: Mapping) -> bool: + """Enable transactional ADP dummy handling while PP remains follow-up.""" + return not mapping.has_pp() def create_py_executor_instance( @@ -2696,8 +2679,12 @@ def create_py_executor_instance( enable_prefix_aware_scheduling=enable_prefix_aware_scheduling, ) - mb_scheduler = BindMicroBatchScheduler(max_batch_size, max_num_tokens, - ctx_chunk_config) + mb_scheduler = BindMicroBatchScheduler( + max_batch_size, + max_num_tokens, + ctx_chunk_config, + no_schedule_until_state=no_schedule_until_state, + ) reorder_policy_config = llm_args.reorder_policy_config if reorder_policy_config is not None: diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index cfb0360d1c80..7e8304c45afd 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -355,11 +355,10 @@ def __init__( self.mapping = mapping if mapping.has_pp(): init_pp_comm(mapping) - # Start with the established pool size. Once the model is loaded we - # selectively enable headroom for the non-PP DeepSeek-V4 overlap path. + # The overlap scheduler can hold two iterations' requests at once. + # Every model-side buffer indexed by py_seq_slot must span this pool. from ._util import (compute_max_num_sequences, - should_enable_dsv4_adp_dummy_fixes, - should_enable_dsv4_overlap_headroom) + should_enable_adp_dummy_fixes) self.max_num_seq_slots = compute_max_num_sequences( mapping, self.batch_size, llm_args.disable_overlap_scheduler) self.dist = dist @@ -433,6 +432,7 @@ def __init__( sparse_attention_config=self.sparse_attention_config, max_num_tokens=self.max_num_tokens, max_seq_len=self.max_seq_len, + max_num_seq_slots=self.max_num_seq_slots, lora_config=lora_config, model_weights_memory_tag=model_weights_memory_tag, model_weights_restore_mode=model_weights_restore_mode, @@ -443,23 +443,11 @@ def __init__( setattr(self, "moe_load_balancer", moe_load_balancer) else: self.model = model - pretrained_config = self.model.model_config.pretrained_config - model_type = getattr(pretrained_config, "model_type", None) - # Keep the scheduler/dummy fix model-scoped, while the larger slot pool - # is restricted to the validated MTP overlap configuration. PP remains - # on its established path for follow-up changes. - self._enable_dsv4_adp_dummy_fixes = (should_enable_dsv4_adp_dummy_fixes( - model_type, mapping)) - self._enable_dsv4_overlap_headroom = ( - should_enable_dsv4_overlap_headroom( - model_type, spec_config, mapping, - llm_args.disable_overlap_scheduler)) - self.max_num_seq_slots = compute_max_num_sequences( - mapping, - self.batch_size, - llm_args.disable_overlap_scheduler, - enable_overlap_headroom=self._enable_dsv4_overlap_headroom, - ) + self._validate_mrope_position_delta_cache_capacity() + # Apply transactional dummy handling to every non-PP disaggregated ADP + # model. Sequence-slot capacity follows the independent overlap + # lifecycle invariant above. + self._enable_adp_dummy_fixes = should_enable_adp_dummy_fixes(mapping) if drafting_loop_wrapper is not None: self.model = drafting_loop_wrapper(self.model) self.model_is_wrapped = True @@ -925,6 +913,33 @@ def set_guided_decoder(self, return success return False + def _validate_mrope_position_delta_cache_capacity(self) -> None: + """Validate slot-indexed MRoPE state on preconstructed models. + + Models created by ModelLoader receive ``max_num_seq_slots`` before + construction. A caller-supplied model bypasses that path, so fail + early instead of indexing past an undersized cache at runtime. + """ + mrope_position_deltas_cache = getattr(self.model, + "mrope_position_deltas_cache", + None) + if mrope_position_deltas_cache is None: + mrope_position_deltas_cache = getattr( + getattr(self.model, "draft_model", None), + "mrope_position_deltas_cache", None) + if mrope_position_deltas_cache is None: + return + + required_size = self.max_num_seq_slots + 1 + actual_size = mrope_position_deltas_cache.shape[0] + if actual_size < required_size: + raise ValueError( + "The supplied model's MRoPE position-delta cache has " + f"{actual_size} slots, but this executor requires at least " + f"{required_size} ({self.max_num_seq_slots} runtime sequence " + "slots plus one reserved dummy slot). Rebuild the model with " + "the executor's sequence-slot capacity.") + @property def use_mrope(self): use_mrope = False @@ -2637,11 +2652,8 @@ def _set_up_spec_metadata( spec_resource_manager: Optional[BaseResourceManager], no_cache=False): spec_config = self.spec_config if self.enable_spec_decode else None - # Only the scoped DeepSeek-V4 overlap path opts into larger metadata - # buffers. Passing None preserves the established max_num_requests - # fallback for every other model, including MTP-Eagle with PP. - num_seq_slots = (self.max_num_seq_slots - if self._enable_dsv4_overlap_headroom else None) + # Slot-indexed metadata must span the same pool as SeqSlotManager. + num_seq_slots = self.max_num_seq_slots if no_cache: return get_spec_metadata( spec_config, @@ -4024,7 +4036,7 @@ def _prepare_tp_inputs( # that carry no MRoPE metadata at all. The cache is zero-initialized and # the write path only ever targets real ``py_seq_slot``s, so this slot # permanently reads back a zero delta. - mrope_dummy_seq_slot = self.max_num_tokens * self.mapping.pp_size + mrope_dummy_seq_slot = self.max_num_seq_slots num_accepted_draft_tokens = [] # per request is_enc_dec = self._is_encoder_decoder_model() cross_encoder_hidden_states: List[torch.Tensor] = [] diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 5d7bd983c8df..0270b0dfc3d7 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -363,7 +363,8 @@ def __init__(self, max_seq_len: Optional[int], lora_config: Optional[LoraConfig] = None, model_weights_memory_tag: Optional[ExecutorMemoryType] = None, - model_weights_restore_mode: Optional[RestoreMode] = None): + model_weights_restore_mode: Optional[RestoreMode] = None, + max_num_seq_slots: Optional[int] = None): """ Initializes the ModelLoader. @@ -379,6 +380,9 @@ def __init__(self, they can be released/materialized independently of buffers. model_weights_restore_mode: RestoreMode for the model weights virtual-memory scope. + max_num_seq_slots: Capacity of model buffers indexed by sequence + slot. This can exceed the scheduler admission batch size when + overlap scheduling is enabled. """ self.llm_args = llm_args self.mapping = mapping @@ -386,6 +390,7 @@ def __init__(self, self.sparse_attention_config = sparse_attention_config self.max_num_tokens = max_num_tokens self.max_seq_len = max_seq_len + self.max_num_seq_slots = max_num_seq_slots self.lora_config = lora_config self.model_weights_memory_tag = model_weights_memory_tag self.model_weights_restore_mode = model_weights_restore_mode @@ -393,6 +398,11 @@ def __init__(self, self._weight_pool_proxy = None self._gms_backend = None + def _set_runtime_model_config_attrs(self, config: ModelConfig) -> None: + """Attach executor-only allocation sizes before model construction.""" + if self.max_num_seq_slots is not None: + config.extra_attrs['max_num_seq_slots'] = self.max_num_seq_slots + @staticmethod def load_config_and_apply_defaults( checkpoint_dir: str, llm_args: TorchLlmArgs, @@ -1402,6 +1412,7 @@ def _load_and_validate_config( load_config_kwargs['model_kwargs'] = self.llm_args.model_kwargs config = checkpoint_loader.load_config(**load_config_kwargs) + self._set_runtime_model_config_attrs(config) # Store nvfp4 config in extra_attrs for Linear layer access config.extra_attrs[ diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 7ce95cbc80c1..1c8f8f36d4b0 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -569,8 +569,8 @@ def __init__( self.resource_manager = resource_manager self.scheduler = scheduler self.model_engine = model_engine - self._enable_dsv4_adp_dummy_fixes = getattr( - model_engine, "_enable_dsv4_adp_dummy_fixes", False) + self._enable_adp_dummy_fixes = getattr(model_engine, + "_enable_adp_dummy_fixes", False) self.enable_attention_dp = model_engine.enable_attention_dp self.dist = dist self.sampler = sampler @@ -3323,7 +3323,7 @@ def _finalize_adp_dummy_allocation(self, can_queue: bool) -> None: must release theirs before retrying or the fixed dummy request ID leaks cache resources on every skipped iteration. """ - if not self._enable_dsv4_adp_dummy_fixes: + if not self._enable_adp_dummy_fixes: return dummy_request = self._pending_adp_dummy_request @@ -5682,16 +5682,15 @@ def _check_disagg_ctx_schedulable_status(self, def _count_schedulable_active_requests(self) -> int: """Count active requests that are ready for scheduling. - The non-PP DeepSeek-V4 disaggregated ADP path mirrors the decoder - scheduler's state window [CONTEXT_INIT, GENERATION_TO_COMPLETE). This - covers generation-first context requests below the lower bound and - terminal requests at the upper bound. Other configurations retain the - established ADP behavior; PP eligibility remains follow-up scope. + The non-PP disaggregated ADP path uses the scheduler's state- + eligibility contract. This keeps decoder-only and encoder-decoder + boundaries and special exclusions aligned without duplicating them + here. PP eligibility remains follow-up scope. Returns: The number of active requests eligible for scheduling. """ - if (not self._enable_dsv4_adp_dummy_fixes + if (not self._enable_adp_dummy_fixes or self.kv_cache_transceiver is None): if self.kv_cache_transceiver is None: return len(self.active_requests) @@ -5701,12 +5700,8 @@ def _count_schedulable_active_requests(self) -> int: if not (req.is_disagg_generation_init_state or req.is_disagg_generation_transmission_in_progress)) - schedule_from_value = LlmRequestState.CONTEXT_INIT.value - to_complete_value = LlmRequestState.GENERATION_TO_COMPLETE.value - - return sum( - 1 for req in self.active_requests - if schedule_from_value <= req.state_value < to_complete_value) + return sum(1 for req in self.active_requests + if self.scheduler.is_request_in_schedulable_state(req)) def _has_adp_dummy_kv_capacity(self, token_nums: Optional[List[int]]) -> bool: @@ -5827,7 +5822,7 @@ def _pad_attention_dp_dummy_request(self): key="attention_dp_dummy_insufficient_kv_capacity") return - if (not self._enable_dsv4_adp_dummy_fixes + if (not self._enable_adp_dummy_fixes or self.kv_cache_transceiver is None): llm_request = self.kv_cache_manager.add_dummy_requests( request_ids=dummy_request_ids, @@ -5862,9 +5857,8 @@ def _pad_attention_dp_dummy_request(self): except OutOfPagesError: dummy_requests = None if not dummy_requests: - logger.warning( - "Cannot allocate DeepSeek-V4 ADP pad dummy; rank schedules " - "an empty batch and the fleet will retry.") + logger.warning("Cannot allocate ADP pad dummy; rank schedules " + "an empty batch and the fleet will retry.") return dummy_request = dummy_requests[0] diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 6149ba5ec0c3..320742235ff5 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -775,14 +775,10 @@ def drafting_loop_wrapper(model): if guided_decoding_config is not None: with allocation_scope(ExecutorMemoryType.GUIDED_DECODER): if mapping.is_last_pp_rank(): - guided_decoder_slots = (max_num_seq_slots if getattr( - model_engine, "_enable_dsv4_overlap_headroom", False) else - max_batch_size) kwargs = { "guided_decoding_config": guided_decoding_config, - # The scoped DeepSeek-V4 path follows the expanded slot - # pool. Other configurations retain max_batch_size. - "max_num_sequences": guided_decoder_slots, + # Guided-decoder state is indexed by sequence slot. + "max_num_sequences": max_num_seq_slots, "vocab_size_padded": model_engine.model.vocab_size_padded, "rank": mapping.rank, } diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py index caa8e3cb3de1..7e9b68d94cd4 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py @@ -216,6 +216,19 @@ def reset_context_requests(self, context_requests: RequestList | None = None) -> class RequestScheduler(ABC): + @property + @abstractmethod + def scheduling_state_range(self) -> tuple[LlmRequestState, LlmRequestState]: + """Return the half-open state range admitted to a forward batch.""" + raise NotImplementedError + + def is_request_in_schedulable_state(self, request: LlmRequest) -> bool: + """Return whether request state permits admission to a forward batch.""" + if is_decoder_context_request_waiting_for_encoder_output(request): + return False + schedule_from, schedule_to = self.scheduling_state_range + return schedule_from.value <= request.state_value < schedule_to.value + @abstractmethod def schedule_request( self, active_requests: RequestList, inflight_request_ids: set[int] @@ -392,10 +405,14 @@ def __init__( max_batch_size: int, max_num_tokens: int = None, ctx_chunk_config: Optional[tuple[StrEnum, int]] = None, + no_schedule_until_state: LlmRequestState = LlmRequestState.CONTEXT_INIT, + no_schedule_after_state: LlmRequestState = LlmRequestState.GENERATION_TO_COMPLETE, ) -> None: super(BindMicroBatchScheduler, self).__init__() self.max_batch_size = max_batch_size self.max_num_tokens = max_num_tokens + self.no_schedule_until_state = no_schedule_until_state + self.no_schedule_after_state = no_schedule_after_state ctx_chunk_config_cpp = None if ctx_chunk_config is not None: @@ -403,7 +420,12 @@ def __init__( ctx_chunk_config[0]._to_pybind(), ctx_chunk_config[1] ) - self.impl = tb_internal.algorithms.MicroBatchScheduler(ctx_chunk_config_cpp, max_num_tokens) + self.impl = tb_internal.algorithms.MicroBatchScheduler( + ctx_chunk_config=ctx_chunk_config_cpp, + max_context_length=max_num_tokens, + no_schedule_until_state=no_schedule_until_state, + no_schedule_after_state=no_schedule_after_state, + ) def schedule( self, active_requests: RequestList, inflight_request_ids: set[int] @@ -427,6 +449,13 @@ def __init__( self.capacity_scheduler = capacity_scheduler self.micro_batch_scheduler = micro_batch_scheduler + @property + def scheduling_state_range(self) -> tuple[LlmRequestState, LlmRequestState]: + return ( + self.micro_batch_scheduler.no_schedule_until_state, + self.micro_batch_scheduler.no_schedule_after_state, + ) + def schedule_request( self, active_requests: RequestList, inflight_request_ids: set[int] ) -> SchedulerOutput: @@ -1867,6 +1896,13 @@ def __init__( no_schedule_until_state=no_schedule_until_state, ) + @property + def scheduling_state_range(self) -> tuple[LlmRequestState, LlmRequestState]: + return ( + self.micro_batch_scheduler.no_schedule_until_state, + self.micro_batch_scheduler.no_schedule_after_state, + ) + def schedule_request( self, active_requests: RequestList, inflight_request_ids: set[int] ) -> SchedulerOutput: diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py index 958afc59ac04..56f353f936c0 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py @@ -205,6 +205,8 @@ def __init__( # MicroBatchScheduler. For encoder-decoder models, caller should pass # no_schedule_until_state=ENCODER_INIT to widen the range (same as # C++ trtEncoderModel which passes kENCODER_INIT). + self.no_schedule_until_state = no_schedule_until_state + self.no_schedule_after_state = no_schedule_after_state self._no_schedule_until_state_value = no_schedule_until_state.value self._no_schedule_after_state_value = no_schedule_after_state.value self._context_init_state_value = LlmRequestState.CONTEXT_INIT.value @@ -220,6 +222,12 @@ def __init__( os.environ.get("TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN", "0") == "1" ) + @property + def scheduling_state_range( + self, + ) -> tuple[LlmRequestState, LlmRequestState]: + return self.no_schedule_until_state, self.no_schedule_after_state + def schedule_request( self, active_requests: RequestList, inflight_request_ids: set[int] ) -> SchedulerOutput: diff --git a/tensorrt_llm/_torch/speculative/interface.py b/tensorrt_llm/_torch/speculative/interface.py index 721b3942ed04..dfad4ecd4257 100644 --- a/tensorrt_llm/_torch/speculative/interface.py +++ b/tensorrt_llm/_torch/speculative/interface.py @@ -562,8 +562,8 @@ class SpecMetadata: # Vocab size used for draft_probs buffer allocation. vocab_size: int = 0 # Size of the SeqSlotManager pool. py_seq_slot values range over - # [0, num_seq_slots); DeepSeek-V4 overlap can use 2 * max_batch_size, - # larger than max_num_requests (== max_batch_size). + # [0, num_seq_slots); overlap can use 2 * max_batch_size, larger than + # max_num_requests (== max_batch_size). # Slot-indexed buffers (draft_probs) must span this full range. # 0 falls back to max_num_requests. num_seq_slots: int = 0 @@ -612,10 +612,10 @@ def prepare_rejection_sampling_buffers(self): return # Slot-indexed buffers span the full SeqSlotManager pool: py_seq_slot - # can range over [0, num_seq_slots), which under DeepSeek-V4 overlap - # exceeds max_num_requests. Fall back to max_num_requests when the pool - # size is unknown (0). One extra scratch row at index ``slot_capacity`` - # absorbs CUDA-graph dummy/padding requests (``py_seq_slot is None``). + # can range over [0, num_seq_slots), which under overlap exceeds + # max_num_requests. Fall back to max_num_requests when the pool size is + # unknown (0). One extra scratch row at index ``slot_capacity`` absorbs + # CUDA-graph dummy/padding requests (``py_seq_slot is None``). slot_capacity = self.num_seq_slots or self.max_num_requests num_slot_rows = slot_capacity + 1 diff --git a/tensorrt_llm/_torch/speculative/utils.py b/tensorrt_llm/_torch/speculative/utils.py index bffa8833058c..22ca1ec0ca03 100644 --- a/tensorrt_llm/_torch/speculative/utils.py +++ b/tensorrt_llm/_torch/speculative/utils.py @@ -87,7 +87,7 @@ def get_spec_metadata(spec_config, use_rejection_sampling = getattr(spec_config, "use_rejection_sampling", False) # Slot-indexed buffers (draft_probs) must span the SeqSlotManager pool; - # DeepSeek-V4 overlap can exceed max_num_requests. + # Overlap can make the sequence-slot pool exceed max_num_requests. num_seq_slots = (num_seq_slots if num_seq_slots is not None else max_num_requests) vocab_size = getattr(model_config, "vocab_size", 0) diff --git a/tests/unittest/_torch/executor/test_benchmark_disagg.py b/tests/unittest/_torch/executor/test_benchmark_disagg.py index d7992e1b445c..470ca9b95441 100644 --- a/tests/unittest/_torch/executor/test_benchmark_disagg.py +++ b/tests/unittest/_torch/executor/test_benchmark_disagg.py @@ -31,7 +31,7 @@ import pytest from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState -from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm._torch.pyexecutor.scheduler import RequestScheduler, ScheduledRequests # --------------------------------------------------------------------------- # Helpers @@ -52,6 +52,8 @@ def _make_active_request( LlmRequestState.DISAGG_TRANS_ERROR if in_error else LlmRequestState.GENERATION_IN_PROGRESS ) req.is_attention_dp_dummy = False + req.is_context_init_state = False + req.py_encoder_output_ready_event = None return req @@ -585,13 +587,24 @@ def __init__( self.max_total_draft_tokens = 0 self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None - self._enable_dsv4_adp_dummy_fixes = True + self._enable_adp_dummy_fixes = True self.max_num_tokens = None self.dist = Mock() self.dist.tp_size = tp_size self.dist.tp_allgather.side_effect = lambda value: [value] + self.scheduler = Mock() + self.scheduler.scheduling_state_range = ( + LlmRequestState.CONTEXT_INIT, + LlmRequestState.GENERATION_TO_COMPLETE, + ) + self.scheduler.is_request_in_schedulable_state.side_effect = ( + lambda request: RequestScheduler.is_request_in_schedulable_state( + self.scheduler, request + ) + ) + self.kv_cache_manager = Mock() self.kv_cache_manager.mapping.has_cp_helix.return_value = False self.kv_cache_manager.get_num_available_tokens.return_value = 1 << 30 diff --git a/tests/unittest/_torch/executor/test_dual_pool_kv_cache.py b/tests/unittest/_torch/executor/test_dual_pool_kv_cache.py index 4782ead014de..5d478a2b45a8 100644 --- a/tests/unittest/_torch/executor/test_dual_pool_kv_cache.py +++ b/tests/unittest/_torch/executor/test_dual_pool_kv_cache.py @@ -865,6 +865,29 @@ def test_cross_kv_cache_manager_and_until_state_are_forwarded(self): impl.assert_called_once_with([], kv_mgr, None, cross_mgr) +class TestBindMicroBatchSchedulerStateRange: + """C++-bound micro-batch scheduling exposes its configured state range.""" + + def test_encoder_state_range_is_forwarded(self): + from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState + from tensorrt_llm._torch.pyexecutor.scheduler.scheduler import BindMicroBatchScheduler + + with patch( + "tensorrt_llm._torch.pyexecutor.scheduler.scheduler.tb_internal.algorithms.MicroBatchScheduler" + ) as micro_cls: + micro_cls.return_value = Mock() + scheduler = BindMicroBatchScheduler( + max_batch_size=8, + max_num_tokens=4096, + no_schedule_until_state=LlmRequestState.ENCODER_INIT, + ) + + kwargs = micro_cls.call_args.kwargs + assert kwargs["no_schedule_until_state"] == LlmRequestState.ENCODER_INIT + assert kwargs["no_schedule_after_state"] == LlmRequestState.GENERATION_TO_COMPLETE + assert scheduler.no_schedule_until_state == LlmRequestState.ENCODER_INIT + + class TestSimpleUnifiedSchedulerCrossParam: """V1 Python ``SimpleUnifiedScheduler`` exposes cross-KV wiring.""" @@ -894,6 +917,10 @@ def test_cross_kv_cache_manager_and_until_state_are_forwarded(self): assert ( scheduler.micro_batch_scheduler.no_schedule_until_state == LlmRequestState.ENCODER_INIT ) + assert scheduler.scheduling_state_range == ( + LlmRequestState.ENCODER_INIT, + LlmRequestState.GENERATION_TO_COMPLETE, + ) # --------------------------------------------------------------------------- diff --git a/tests/unittest/_torch/executor/test_model_loader_gms.py b/tests/unittest/_torch/executor/test_model_loader_gms.py index 5b20f4f81653..a12151873793 100644 --- a/tests/unittest/_torch/executor/test_model_loader_gms.py +++ b/tests/unittest/_torch/executor/test_model_loader_gms.py @@ -156,6 +156,16 @@ def _build_source_identity(_cls, *_args, **kwargs): return loader +def test_runtime_model_config_attrs_include_sequence_slot_capacity(): + loader = object.__new__(ModelLoader) + loader.max_num_seq_slots = 16 + config = SimpleNamespace(extra_attrs={}) + + loader._set_runtime_model_config_attrs(config) + + assert config.extra_attrs["max_num_seq_slots"] == 16 + + def _build_gms_backend(*, is_rw, events): backend = MagicMock() backend.connect.return_value = True diff --git a/tests/unittest/_torch/executor/test_py_executor.py b/tests/unittest/_torch/executor/test_py_executor.py index 50f33d321c2f..914ab1251a61 100644 --- a/tests/unittest/_torch/executor/test_py_executor.py +++ b/tests/unittest/_torch/executor/test_py_executor.py @@ -30,6 +30,7 @@ from tensorrt_llm._torch.pyexecutor.resource_manager import NoFreeSlotsError, ResourceManagerType from tensorrt_llm._torch.pyexecutor.scheduler import ( FCFSWaitingQueue, + RequestScheduler, ScheduledRequests, SerializableSchedulerOutput, ) @@ -1290,6 +1291,8 @@ def _make_adp_request( req.is_attention_dp_dummy = False req.llm_request_type = llm_request_type req.py_seq_slot = None + req.is_context_init_state = state == LlmRequestState.CONTEXT_INIT + req.py_encoder_output_ready_event = None return req @@ -1304,7 +1307,7 @@ def __init__( kv_manager_max_seq_len=None, is_warmup=False, benchmark_req_queues_size=0, - enable_dsv4_adp_dummy_fixes=True, + enable_adp_dummy_fixes=True, ): self.enable_attention_dp = enable_attention_dp self.kv_cache_transceiver = kv_cache_transceiver @@ -1318,7 +1321,7 @@ def __init__( self.max_num_tokens = max_num_tokens self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None - self._enable_dsv4_adp_dummy_fixes = enable_dsv4_adp_dummy_fixes + self._enable_adp_dummy_fixes = enable_adp_dummy_fixes self.add_dummy_calls = [] self.model_engine = Mock(max_num_tokens=max_num_tokens, max_seq_len=max_seq_len) @@ -1326,6 +1329,17 @@ def __init__( self.dist.tp_size = 1 self.dist.tp_allgather.side_effect = lambda value: [value] + self.scheduler = Mock() + self.scheduler.scheduling_state_range = ( + LlmRequestState.CONTEXT_INIT, + LlmRequestState.GENERATION_TO_COMPLETE, + ) + self.scheduler.is_request_in_schedulable_state.side_effect = ( + lambda request: RequestScheduler.is_request_in_schedulable_state( + self.scheduler, request + ) + ) + kv_cache_manager = Mock() kv_cache_manager.mapping.has_cp_helix.return_value = False kv_cache_manager.get_num_available_tokens.return_value = 1 << 30 @@ -1444,9 +1458,9 @@ def test_adp_dummy_role_unchanged_when_attention_dp_disabled(): LlmRequestState.DISAGG_CONTEXT_WAIT_SCHEDULER, ], ) -def test_disabled_dsv4_gate_preserves_existing_disagg_behavior(state): - # The disabled gate covers non-DSv4 and PP configurations. - stub = _StubADPExecutor(enable_dsv4_adp_dummy_fixes=False) +def test_disabled_adp_dummy_fix_gate_preserves_pp_behavior(state): + # PP configurations remain on the established dummy path. + stub = _StubADPExecutor(enable_adp_dummy_fixes=False) stub.active_requests = [_make_adp_request(state)] stub.expected_num_active_requests = 1 @@ -1487,6 +1501,71 @@ def test_pad_dummy_added_when_only_wait_scheduler_requests_disagg(): assert len(stub.active_requests) == 2 +def test_encoder_init_uses_encoder_decoder_scheduler_state_window(): + stub = _StubADPExecutor() + stub.scheduler.scheduling_state_range = ( + LlmRequestState.ENCODER_INIT, + LlmRequestState.GENERATION_TO_COMPLETE, + ) + stub.active_requests = [_make_adp_request(LlmRequestState.ENCODER_INIT)] + stub.expected_num_active_requests = 1 + + _run_pad(stub) + + assert stub.add_dummy_calls == [] + assert len(stub.active_requests) == 1 + + +def test_decoder_context_waiting_for_encoder_output_is_not_counted(): + stub = _StubADPExecutor() + request = _make_adp_request(LlmRequestState.CONTEXT_INIT) + request.py_encoder_output_ready_event = Mock() + request.py_encoder_output_ready_event.query.return_value = False + stub.active_requests = [request] + stub.expected_num_active_requests = 2 + + _run_pad(stub) + + assert len(stub.add_dummy_calls) == 1 + assert len(stub.active_requests) == 2 + + +def test_non_dsv4_disagg_adp_mixed_rank_states_stay_queueable(): + # The generic non-PP path must give both ranks a non-empty scheduled batch: + # one rank schedules its real request, while the terminal-only rank + # schedules the dummy inserted for the scheduler-excluded request. + busy_rank = _StubADPExecutor() + busy_rank.active_requests = [_make_adp_request(_STATE_GENERATION_IN_PROGRESS)] + busy_rank.expected_num_active_requests = 2 + terminal_rank = _StubADPExecutor() + terminal_rank.active_requests = [_make_adp_request(_STATE_GENERATION_TO_COMPLETE)] + terminal_rank.expected_num_active_requests = 2 + + _run_pad(busy_rank) + _run_pad(terminal_rank) + + assert busy_rank.add_dummy_calls == [] + assert len(terminal_rank.add_dummy_calls) == 1 + rank_batch_sizes = [ + busy_rank._count_schedulable_active_requests(), + terminal_rank._count_schedulable_active_requests(), + ] + assert rank_batch_sizes == [1, 1] + + for stub, batch_size in zip((busy_rank, terminal_rank), rank_batch_sizes, strict=True): + stub.dist.tp_allgather.side_effect = None + stub.dist.tp_allgather.return_value = rank_batch_sizes + can_queue, can_queue_this_rank = PyExecutor._can_queue( + stub, types.SimpleNamespace(batch_size=batch_size) + ) + + assert can_queue is True + assert can_queue_this_rank is True + PyExecutor._finalize_adp_dummy_allocation(stub, can_queue) + + assert terminal_rank._pending_adp_dummy_request is None + + def test_pad_dummy_allocation_failure_skips_padding(): # add_dummy_requests returns None when the rank has no free cache # resources for even a 1-token dummy (possible while non-schedulable @@ -1504,20 +1583,7 @@ def test_pad_dummy_allocation_failure_skips_padding(): assert not any(r.is_attention_dp_dummy for r in stub.active_requests) -def test_disabled_dsv4_gate_checks_full_generation_capacity(): - stub = _StubADPExecutor(enable_dsv4_adp_dummy_fixes=False) - stub.max_total_draft_tokens = 4 - stub.kv_cache_manager.get_num_available_tokens.return_value = 4 - - _run_pad(stub) - - stub.kv_cache_manager.get_num_available_tokens.assert_called_once_with( - token_num_upper_bound=5, max_num_draft_tokens=4 - ) - stub.kv_cache_manager.add_dummy_requests.assert_not_called() - - -def test_dsv4_pad_dummy_checks_full_context_capacity(): +def test_adp_pad_dummy_checks_full_context_capacity(): stub = _StubADPExecutor(max_num_tokens=4096) stub._adp_dummy_is_gen = False stub.kv_cache_manager.get_num_available_tokens.return_value = 1024 @@ -1531,7 +1597,7 @@ def test_dsv4_pad_dummy_checks_full_context_capacity(): assert stub._pending_adp_dummy_request is None -def test_dsv4_pad_dummy_checks_full_generation_capacity(): +def test_adp_pad_dummy_checks_full_generation_capacity(): stub = _StubADPExecutor() stub.kv_cache_manager.get_num_available_tokens.return_value = 0 @@ -1544,7 +1610,7 @@ def test_dsv4_pad_dummy_checks_full_generation_capacity(): assert stub._pending_adp_dummy_request is None -def test_dsv4_pad_dummy_capacity_includes_draft_reserve(): +def test_adp_pad_dummy_capacity_includes_draft_reserve(): stub = _StubADPExecutor() stub.max_total_draft_tokens = 3 stub.kv_cache_manager.get_num_available_tokens.return_value = 3 diff --git a/tests/unittest/_torch/executor/test_pytorch_model_engine.py b/tests/unittest/_torch/executor/test_pytorch_model_engine.py index 884fb8e2eebb..68a86848fdec 100644 --- a/tests/unittest/_torch/executor/test_pytorch_model_engine.py +++ b/tests/unittest/_torch/executor/test_pytorch_model_engine.py @@ -1562,6 +1562,7 @@ def test_prepare_tp_inputs_with_partial_mrope_segments(self) -> None: attn_metadata.is_cuda_graph = False model_engine.max_num_tokens = 32 + model_engine.max_num_seq_slots = 8 model_engine.input_ids_cuda = torch.zeros(32, dtype=torch.int32, device='cuda') @@ -1593,6 +1594,9 @@ def test_prepare_tp_inputs_with_partial_mrope_segments(self) -> None: dummy_request.sampling_config.beam_width = 1 dummy_request.py_multimodal_data = {} dummy_request.is_cuda_graph_dummy = True + dummy_request.py_mrope_position_delta = torch.tensor([[0]], + dtype=torch.int32, + device='cuda') scheduled_requests = ScheduledRequests() scheduled_requests.context_requests_last_chunk = [] @@ -1615,10 +1619,10 @@ def test_prepare_tp_inputs_with_partial_mrope_segments(self) -> None: [0]) # Read slots are dense w.r.t. the generation batch: the padded dummy # has no MRoPE metadata, so it resolves to the reserved zero slot - # (max_num_tokens * pp_size) rather than being dropped, which would + # (max_num_seq_slots) rather than being dropped, which would # shift every later request onto another request's delta. self.assertEqual(result["mrope_delta_read_seq_slots"].cpu().tolist(), - [0, 32]) + [0, model_engine.max_num_seq_slots]) self.assertNotIn("multimodal_embedding", multimodal_request.py_multimodal_data) kv_cache_manager.shutdown() @@ -1713,11 +1717,11 @@ def test_prepare_tp_inputs_mixed_text_only_keeps_mrope_deltas_dense( kv_cache_manager=kv_cache_manager, attn_metadata=attn_metadata) - # One entry per generation request, in batch order. Slot 32 is the - # reserved zero slot (max_num_tokens * pp_size) standing in for the - # text-only request's zero delta. + # One entry per generation request, in batch order. The reserved zero + # slot (max_num_seq_slots) stands in for the text-only request's zero + # delta. self.assertEqual(result["mrope_delta_read_seq_slots"].cpu().tolist(), - [0, 32, 2]) + [0, model_engine.max_num_seq_slots, 2]) # Only the two multimodal requests seed the seq-slot delta cache. self.assertEqual(result["mrope_delta_write_seq_slots"].cpu().tolist(), [0, 2]) @@ -1823,6 +1827,16 @@ def test_promoted_mrope_context_uses_decode_state_contract(self) -> None: self.assertEqual(model_engine.previous_request_ids, []) kv_cache_manager.shutdown() + def test_preconstructed_mrope_model_requires_runtime_seq_slot_capacity( + self) -> None: + model_engine = object.__new__(PyTorchModelEngine) + model_engine.max_num_seq_slots = 8 + model_engine.model = SimpleNamespace( + mrope_position_deltas_cache=torch.zeros(8, dtype=torch.int32)) + + with self.assertRaisesRegex(ValueError, "requires at least 9"): + model_engine._validate_mrope_position_delta_cache_capacity() + def test_kv_cache_manager_with_execution_stream(self) -> None: """Test that KVCacheManager uses the provided execution_stream. """ diff --git a/tests/unittest/_torch/executor/test_seq_slot_sizing.py b/tests/unittest/_torch/executor/test_seq_slot_sizing.py index d42e6f4483c9..2ffced3e42c4 100644 --- a/tests/unittest/_torch/executor/test_seq_slot_sizing.py +++ b/tests/unittest/_torch/executor/test_seq_slot_sizing.py @@ -1,88 +1,48 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""DeepSeek-V4 seq-slot sizing includes overlap headroom. +"""Seq-slot pool and slot-indexed state include overlap headroom. Under the overlap scheduler, requests finished in the previous iteration still hold their sequence slots when the next iteration's prepare_resources runs, while the V2 scheduler has already dropped them from its budget (no_schedule_after_state=GENERATION_TO_COMPLETE) and backfilled their seats. Transient slot demand is therefore -2 * max_batch_size. The headroom is intentionally limited to DeepSeek-V4; -other models preserve their established sizing pending separate validation. +2 * max_batch_size for every non-PP overlap configuration. compute_max_num_sequences is the single sizing implementation used both for the executor's SeqSlotManager pool (create_py_executor_instance) and for the sampler state (create_torch_sampler_args). """ -from unittest.mock import Mock - import pytest from tensorrt_llm._torch.pyexecutor._util import ( compute_max_num_sequences, create_torch_sampler_args, - should_enable_dsv4_adp_dummy_fixes, - should_enable_dsv4_overlap_headroom, + should_enable_adp_dummy_fixes, ) from tensorrt_llm.mapping import Mapping SIZING_CASES = [ - # (pp_size, disable_overlap, enable_overlap_headroom, expected_factor) - (1, False, True, 2), - (1, False, False, 1), - (1, True, True, 1), - # Existing PP sizing is preserved regardless of the DSv4 opt-in. - (2, False, True, 2), - (4, False, True, 4), - (4, True, False, 4), + # (pp_size, disable_overlap, expected_factor) + (1, False, 2), + (1, True, 1), + # PP already sizes the pool for its micro-batch count. + (2, False, 2), + (4, False, 4), + (4, True, 4), ] -@pytest.mark.parametrize( - "model_type,has_spec,is_mtp_one_model,pp_size,disable_overlap,expected", - [ - ("deepseek_v4", True, True, 1, False, True), - ("deepseek_v3", True, True, 1, False, False), - ("deepseek_v4", False, False, 1, False, False), - ("deepseek_v4", True, False, 1, False, False), - ("deepseek_v4", True, True, 2, False, False), - ("deepseek_v4", True, True, 1, True, False), - ], -) -def test_dsv4_overlap_headroom_gate( - model_type, has_spec, is_mtp_one_model, pp_size, disable_overlap, expected -): - spec_config = None - if has_spec: - spec_config = Mock() - spec_config.spec_dec_mode.is_mtp_eagle_one_model.return_value = is_mtp_one_model +@pytest.mark.parametrize("pp_size,expected", [(1, True), (2, False)]) +def test_adp_dummy_fix_gate(pp_size, expected): mapping = Mapping(world_size=pp_size, tp_size=1, pp_size=pp_size) - - assert ( - should_enable_dsv4_overlap_headroom(model_type, spec_config, mapping, disable_overlap) - is expected - ) + assert should_enable_adp_dummy_fixes(mapping) is expected -@pytest.mark.parametrize( - "model_type,pp_size,expected", - [ - ("deepseek_v4", 1, True), - ("deepseek_v3", 1, False), - ("deepseek_v4", 2, False), - ], -) -def test_dsv4_adp_dummy_fix_gate(model_type, pp_size, expected): - mapping = Mapping(world_size=pp_size, tp_size=1, pp_size=pp_size) - assert should_enable_dsv4_adp_dummy_fixes(model_type, mapping) is expected - - -@pytest.mark.parametrize( - "pp_size,disable_overlap,enable_overlap_headroom,expected_factor", SIZING_CASES -) -def test_compute_max_num_sequences_scopes_overlap_headroom( - pp_size, disable_overlap, enable_overlap_headroom, expected_factor +@pytest.mark.parametrize("pp_size,disable_overlap,expected_factor", SIZING_CASES) +def test_compute_max_num_sequences_includes_overlap_headroom( + pp_size, disable_overlap, expected_factor ): max_batch_size = 8 mapping = Mapping(world_size=pp_size, tp_size=1, pp_size=pp_size) @@ -91,26 +51,26 @@ def test_compute_max_num_sequences_scopes_overlap_headroom( mapping, max_batch_size, disable_overlap, - enable_overlap_headroom=enable_overlap_headroom, ) == max_batch_size * expected_factor ) -@pytest.mark.parametrize("slot_factor", [1, 2]) -def test_sampler_uses_executor_slot_pool_capacity(slot_factor): +@pytest.mark.parametrize("pp_size,disable_overlap,expected_factor", SIZING_CASES) +def test_sampler_uses_executor_slot_pool_capacity(pp_size, disable_overlap, expected_factor): max_batch_size = 8 - mapping = Mapping(world_size=1, tp_size=1, pp_size=1) - max_num_sequences = max_batch_size * slot_factor + mapping = Mapping(world_size=pp_size, tp_size=1, pp_size=pp_size) args = create_torch_sampler_args( mapping, max_seq_len=1024, max_batch_size=max_batch_size, speculative_config=None, max_beam_width=1, - disable_overlap_scheduler=False, + disable_overlap_scheduler=disable_overlap, enable_async_worker=False, enable_speculative_beam_history_d2h=False, - max_num_sequences=max_num_sequences, ) - assert args.max_num_sequences == max_num_sequences + assert args.max_num_sequences == compute_max_num_sequences( + mapping, max_batch_size, disable_overlap + ) + assert args.max_num_sequences == max_batch_size * expected_factor diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen2_5vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen2_5vl.py index 3f6bc50706ac..f3481707dbdc 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen2_5vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen2_5vl.py @@ -24,7 +24,8 @@ Qwen2VLHfWeightMapper from tensorrt_llm._torch.models.modeling_qwen2vl import ( Qwen2_5_VLModel, Qwen2VisionModelBase, Qwen2VLInputProcessorBase, - Qwen2VLModel, _prepare_qwen_vl_mrope_config) + Qwen2VLModel, _get_mrope_position_delta_cache_size, + _prepare_qwen_vl_mrope_config) from tensorrt_llm._torch.models.modeling_qwen3vl import \ Qwen3VLInputProcessorBase from tensorrt_llm._utils import get_sm_version @@ -426,6 +427,13 @@ def _mrope_param(delta: int) -> MultimodalParams: }) +def test_mrope_delta_cache_size_uses_runtime_seq_slot_capacity(): + model_config = ModelConfig(max_num_tokens=32) + model_config.extra_attrs['max_num_seq_slots'] = 8 + + assert _get_mrope_position_delta_cache_size(model_config) == 9 + + def test_prepare_qwen_vl_mrope_config_mixed_context_generation(): rotary_dim = 2 num_tokens = 5 diff --git a/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py b/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py index a71b9f1e6640..f24fce926b65 100644 --- a/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py +++ b/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py @@ -67,7 +67,7 @@ def test_prepare_buffers_allocates_full_draft_probs_on_vocab_mismatch(): def test_prepare_buffers_span_seq_slot_pool(): - # Under DeepSeek-V4 overlap scheduling the SeqSlotManager pool + # Under overlap scheduling the SeqSlotManager pool # (num_seq_slots) can exceed max_num_requests; py_seq_slot then indexes past # max_num_requests. Slot-indexed buffers must span the full pool plus the # dummy scratch row, and dummy_slot_row must land on that last row so a real