diff --git a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py index c2a6a5bb4de1..2d876b2a9b78 100644 --- a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py @@ -519,8 +519,6 @@ def replay(self, key: KeyType, def _get_padded_batch(self, batch: ScheduledRequests, resource_manager: ResourceManager, runtime_draft_len: int) -> int: - kv_cache_manager = resource_manager.get_resource_manager( - self.config.kv_cache_manager_key) can_run_cuda_graph = batch.can_run_cuda_graph batch_size = batch.batch_size new_batch_size = batch_size @@ -539,15 +537,12 @@ def _get_padded_batch(self, batch: ScheduledRequests, or new_batch_size > self.max_supported_batch_size): return 0 - # When dynamic draft length is enabled (one-model path), we treat the determined runtime draft length - # as the source of truth and pad the batch size up to the nearest existing graph - # for that draft length. - if (self.spec_config and self.spec_config.draft_len_schedule - and self.spec_config.spec_dec_mode.support_dynamic_draft_len()): - padded_batch_size = self._round_up_batch_size_with_draft_len( - new_batch_size, runtime_draft_len) - else: - padded_batch_size = self._round_up_batch_size(new_batch_size) + # Pad the batch size up to the nearest existing graph for the runtime + # draft length. With dynamic draft length (one-model path) the runtime + # draft length is the source of truth for which graphs are eligible; + # otherwise this reduces to plain batch-size rounding. + padded_batch_size = self._round_up_batch_size_with_draft_len( + new_batch_size, runtime_draft_len) if batch_size == padded_batch_size: return 0 @@ -558,66 +553,148 @@ def _get_padded_batch(self, batch: ScheduledRequests, if padding_size + batch.batch_size > self.config.batch_size: return 0 - runtime_tokens_per_gen_step = ( - self.spec_config.get_runtime_tokens_per_gen_step(runtime_draft_len) - if self.spec_config is not None else 1 + runtime_draft_len) - runtime_draft_token_buffer_width = runtime_tokens_per_gen_step - 1 - # No padding if it would create too many concurrent requests. # This is not strictly required, but we should probably # respect the requirement just in case that changes in the future. # Use per-draft-len dummy requests for dynamic draft length support. - if runtime_draft_len not in self.padding_dummy_requests: - dummy_encoder_output_len = None - if self.is_encoder_decoder: - cross_kv_cache_manager = resource_manager.get_resource_manager( - ResourceManagerType.CROSS_KV_CACHE_MANAGER) - if cross_kv_cache_manager is None: - return 0 - dummy_encoder_output_len = self._get_padding_dummy_encoder_output_len( - cross_kv_cache_manager) - - # Get draft KV cache manager only for one-model speculative decoding. - # In two-model mode, each model has its own KV cache manager, so - # draft_kv_cache_manager should be None. - draft_kv_cache_manager = get_draft_kv_cache_manager( - self.spec_config, resource_manager) - - # Use unique dummy request ID per draft length - dummy_request_id = CUDA_GRAPH_DUMMY_REQUEST_ID - runtime_draft_len - dummy_request = kv_cache_manager.add_dummy_requests( - [dummy_request_id], - token_nums=[ENC_DEC_CUDA_GRAPH_DUMMY_TOKEN_NUM] - if self.is_encoder_decoder else None, - is_gen=True, - max_num_draft_tokens=runtime_draft_token_buffer_width, - use_mrope=self.config.use_mrope, - max_beam_width=self.config.max_beam_width, - encoder_output_lens=[dummy_encoder_output_len] - if dummy_encoder_output_len is not None else None, - draft_kv_cache_manager=draft_kv_cache_manager) - - if dummy_request is None: - return 0 - else: - dummy_request = dummy_request[0] - dummy_request.is_cuda_graph_dummy = True - if self.is_encoder_decoder: - if not self._add_cross_dummy_request( - dummy_request, resource_manager, - dummy_encoder_output_len, draft_kv_cache_manager): - return 0 - - spec_res_mgr = resource_manager.get_resource_manager( - ResourceManagerType.SPEC_RESOURCE_MANAGER) - if spec_res_mgr: - spec_res_mgr.add_dummy_requests([dummy_request_id]) - self.padding_dummy_requests[runtime_draft_len] = dummy_request - - padding_dummy_request = self.padding_dummy_requests[runtime_draft_len] + padding_dummy_request = self._get_or_create_padding_dummy( + resource_manager, runtime_draft_len) + if padding_dummy_request is None: + logger.warning_once( + "Failed to allocate the CUDA graph padding dummy request " + f"(draft_len={runtime_draft_len}) from a saturated KV cache; " + "falling back to eager mode for padded batches.", + key=f"cuda_graph_padding_dummy_fallback_{runtime_draft_len}") + return 0 + batch.generation_requests.extend([padding_dummy_request] * padding_size) return padding_size + def _get_or_create_padding_dummy( + self, resource_manager: ResourceManager, + runtime_draft_len: int) -> Optional[LlmRequest]: + """Returns the padding dummy request for the given draft length, + allocating it from the KV cache manager on first use. + + Returns None when the KV cache manager cannot allocate the dummy + (e.g. saturated cache); the batch cannot be padded until a retry + succeeds. + """ + if runtime_draft_len in self.padding_dummy_requests: + return self.padding_dummy_requests[runtime_draft_len] + + kv_cache_manager = resource_manager.get_resource_manager( + self.config.kv_cache_manager_key) + + runtime_tokens_per_gen_step = ( + self.spec_config.get_runtime_tokens_per_gen_step(runtime_draft_len) + if self.spec_config is not None else 1 + runtime_draft_len) + runtime_draft_token_buffer_width = runtime_tokens_per_gen_step - 1 + + dummy_encoder_output_len = None + if self.is_encoder_decoder: + cross_kv_cache_manager = resource_manager.get_resource_manager( + ResourceManagerType.CROSS_KV_CACHE_MANAGER) + if cross_kv_cache_manager is None: + return None + dummy_encoder_output_len = self._get_padding_dummy_encoder_output_len( + cross_kv_cache_manager) + + # Get draft KV cache manager only for one-model speculative decoding. + # In two-model mode, each model has its own KV cache manager, so + # draft_kv_cache_manager should be None. + draft_kv_cache_manager = get_draft_kv_cache_manager( + self.spec_config, resource_manager) + + # Use unique dummy request ID per draft length + dummy_request_id = CUDA_GRAPH_DUMMY_REQUEST_ID - runtime_draft_len + dummy_request = kv_cache_manager.add_dummy_requests( + [dummy_request_id], + token_nums=[ENC_DEC_CUDA_GRAPH_DUMMY_TOKEN_NUM] + if self.is_encoder_decoder else None, + is_gen=True, + max_num_draft_tokens=runtime_draft_token_buffer_width, + use_mrope=self.config.use_mrope, + max_beam_width=self.config.max_beam_width, + encoder_output_lens=[dummy_encoder_output_len] + if dummy_encoder_output_len is not None else None, + draft_kv_cache_manager=draft_kv_cache_manager) + + if dummy_request is None: + return None + dummy_request = dummy_request[0] + dummy_request.is_cuda_graph_dummy = True + if self.is_encoder_decoder: + if not self._add_cross_dummy_request( + dummy_request, resource_manager, dummy_encoder_output_len, + draft_kv_cache_manager): + return None + + spec_res_mgr = resource_manager.get_resource_manager( + ResourceManagerType.SPEC_RESOURCE_MANAGER) + if spec_res_mgr: + spec_res_mgr.add_dummy_requests([dummy_request_id]) + self.padding_dummy_requests[runtime_draft_len] = dummy_request + return dummy_request + + def _can_pad_any_batch(self, runtime_draft_len: int) -> bool: + """Returns True when _get_padded_batch can pad at least one feasible + batch size for the given draft length (mirrors its rounding and + capacity guards). For example, with graph batch sizes [1, 2, 4] and + max batch size 1, every batch already matches a graph size and no + padding dummy is ever needed. + """ + max_unpadded_batch_size = min(self.config.batch_size, + self.max_supported_batch_size) + for batch_size in range(1, max_unpadded_batch_size + 1): + padded = self._round_up_batch_size_with_draft_len( + batch_size, runtime_draft_len) + if batch_size < padded <= self.config.batch_size: + return True + return False + + def preallocate_padding_dummies(self, + resource_manager: ResourceManager) -> None: + """Eagerly allocates the padding dummies while the KV cache still has + free blocks (called at the end of ModelEngine.warmup, after graph + capture). + + _get_padded_batch otherwise allocates the dummies lazily at the first + padded step; when the KV cache is already saturated by then, the + allocation fails on every step and padded batches permanently fall + back to eager mode. + + Only the draft lengths of captured graphs are preallocated — those are + exactly the ones runtime padding requests — and only when padding can + actually occur for that draft length. Anything else would permanently + hold KV blocks (and spec/hybrid cache slots) that the lazy path never + consumes, regressing tightly-sized deployments. + """ + if not (self.enabled and self.padding_enabled): + return + kv_cache_manager = resource_manager.get_resource_manager( + self.config.kv_cache_manager_key) + if kv_cache_manager is None or getattr(kv_cache_manager, + 'is_estimating_kv_cache', False): + # The estimation-phase KV cache is sized with no headroom for + # retained dummies; holding blocks there can leave the estimation + # requests unschedulable. That executor is discarded anyway, so + # preallocation only matters for the final one. + return + for draft_len in sorted({key[1] for key in self.graphs}): + if not self._can_pad_any_batch(draft_len): + continue + if self._get_or_create_padding_dummy(resource_manager, + draft_len) is None: + logger.warning( + "Could not pre-allocate the CUDA graph padding dummy " + f"request (draft_len={draft_len}) at warmup; allocation " + "will be retried at the first padded step.") + else: + logger.info( + "Pre-allocated the CUDA graph padding dummy request " + f"(draft_len={draft_len}) at warmup.") + def _add_cross_dummy_request( self, dummy_request: LlmRequest, resource_manager: ResourceManager, encoder_output_len: int, @@ -667,9 +744,14 @@ def _round_up_batch_size(self, batch_size: int) -> int: def _round_up_batch_size_with_draft_len(self, batch_size: int, draft_len: int) -> int: - """Finds the smallest graph batch size >= batch_size that also matches the given draft_len.""" + """Finds the smallest graph batch size >= batch_size that also matches the given draft_len. + + The dynamic draft length mapping exists exactly when the dynamic draft + length feature is active (see _compute_dynamic_draft_len_mapping); + without it, this ignores draft_len and reduces to plain batch-size + rounding. + """ if not self.dynamic_draft_len_mapping: - # Fallback to regular round up if no mapping return self._round_up_batch_size(batch_size) start_idx = bisect.bisect_left(self.supported_batch_sizes, batch_size) diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py index 518c7b711162..12c0f3d0f4de 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py @@ -767,6 +767,7 @@ def __init__( is_disagg: bool = False, enable_stats: bool = False, num_reserved_index_slots: int = 1, + is_estimating_kv_cache: bool = False, **kwargs, ) -> None: self.mapping = mapping @@ -789,6 +790,13 @@ def __init__( layer_mask=layer_mask, ) self.is_draft = is_draft + + # Retained so consumers (e.g. CUDAGraphRunner.preallocate_padding_dummies) + # can distinguish the throwaway estimation-phase managers from the + # final ones: the estimation cache is sized with no headroom for + # retained dummy requests. + self.is_estimating_kv_cache = is_estimating_kv_cache + # Set True by a compression manager; generation-step resize then leaves history untouched. self.kv_compression_manages_history: bool = False self.enable_swa_scratch_reuse = ( diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 5afc55544c8c..34c06ec5e9b5 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -1226,6 +1226,13 @@ def warmup(self, resource_manager: ResourceManager) -> None: self._general_warmup(resource_manager, warmup_requests_configs) log_mem_snapshot("warmup/after_memory_pool_prepop") + # Allocate the CUDA graph padding dummies now, while the KV cache is + # empty. Waiting for the first padded step can race KV saturation: + # once the cache is full, the lazy allocation in _get_padded_batch + # fails every step and padded batches silently run eager. + self.cuda_graph_runner.preallocate_padding_dummies(resource_manager) + log_mem_snapshot("warmup/after_preallocate_padding_dummies") + def _warmup_dg_paged_mqa_logits_metadata(self) -> None: """Pre-compile DeepGEMM's `get_paged_mqa_logits_metadata` helper for every 32-aligned batch bucket the runtime can produce. diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index b4faf969d0da..290326c19a9d 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -318,6 +318,11 @@ def __init__( layer_mask=layer_mask, ) self.is_draft = is_draft + # Retained so consumers (e.g. CUDAGraphRunner.preallocate_padding_dummies) + # can distinguish the throwaway estimation-phase managers from the + # final ones: the estimation cache is sized with no headroom for + # retained dummy requests. + self.is_estimating_kv_cache = is_estimating_kv_cache self.num_local_layers = len(self.pp_layers) self.layer_offsets = { idx: offset diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index a1f4f82869bd..a6bd80ef0e66 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -130,7 +130,13 @@ def free_resources(self, request: LlmRequest): def add_dummy_requests(self, request_ids: List[int]): for rid in request_ids: - self.slot_manager.add_slot(rid) + # In two-model speculation the target and draft engines share this + # resource manager, and each registers its own padding dummy under + # the same draft-length-derived request ID. Reserve the slot once + # and share it between them; dummy outputs are discarded, so the + # slot contents never matter (mirrors SuffixAutomatonManager). + if self.slot_manager.get_slot(rid) is None: + self.slot_manager.add_slot(rid) if self.sa_manager is not None: self.sa_manager.add_dummy_requests(request_ids) diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index 3dc12a669aa6..c89e61ce276e 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -104,7 +104,13 @@ def free_resources(self, request: LlmRequest): def add_dummy_requests(self, request_ids: List[int]): for rid in request_ids: - self.slot_manager.add_slot(rid) + # The target and draft engines share this resource manager and + # each registers its padding dummy under the same + # draft-length-derived request ID. Reserve the slot once and share + # it between them; dummy outputs are discarded, so the slot + # contents never matter (mirrors SuffixAutomatonManager). + if self.slot_manager.get_slot(rid) is None: + self.slot_manager.add_slot(rid) if self.sa_manager is not None: self.sa_manager.add_dummy_requests(request_ids) diff --git a/tests/unittest/_torch/executor/test_pytorch_model_engine.py b/tests/unittest/_torch/executor/test_pytorch_model_engine.py index 6347275fd640..b29a9c9f744c 100644 --- a/tests/unittest/_torch/executor/test_pytorch_model_engine.py +++ b/tests/unittest/_torch/executor/test_pytorch_model_engine.py @@ -114,18 +114,21 @@ def create_model_engine_and_kvcache(llm_args: TorchLlmArgs = None, num_layers = 1 batch_size = 13 - llm_args = llm_args if llm_args else TorchLlmArgs( - model="dummy", - max_batch_size=batch_size, - max_num_tokens=max_tokens, - cuda_graph_config=CudaGraphConfig( - enable_padding=True, batch_sizes=[1, 2, 4, 8, 16, 32, 64, 128])) - test_batches = (5, 13) - for batch_size in test_batches: - assert batch_size not in llm_args.cuda_graph_config.batch_sizes + if llm_args is None: + llm_args = TorchLlmArgs(model="dummy", + max_batch_size=batch_size, + max_num_tokens=max_tokens, + cuda_graph_config=CudaGraphConfig( + enable_padding=True, + batch_sizes=[1, 2, 4, 8, 16, 32, 64, 128])) + # The padding tests below rely on these properties of the default + # config: batches of 5 and 13 must round up to 8 and 16 respectively. + test_batches = (5, 13) + for test_batch_size in test_batches: + assert test_batch_size not in llm_args.cuda_graph_config.batch_sizes - assert (8 in llm_args.cuda_graph_config.batch_sizes - and 16 in llm_args.cuda_graph_config.batch_sizes) + assert (8 in llm_args.cuda_graph_config.batch_sizes + and 16 in llm_args.cuda_graph_config.batch_sizes) model_engine = DummyModelEngine(llm_args, torch.half) @@ -378,12 +381,92 @@ def test_warmup(self): num_free_before = kv_cache_manager.get_num_free_blocks() model_engine.warmup(resource_manager) - # Make sure we don't leak any blocks. + + # Warmup pre-allocates the CUDA graph padding dummy for each captured + # draft length (only draft_len 0 here, no speculation), which + # intentionally keeps holding its KV blocks so padding cannot fall + # back to eager once the cache saturates. + padding_dummies = model_engine.cuda_graph_runner.padding_dummy_requests + self.assertEqual([0], sorted(padding_dummies.keys())) + + # Make sure we don't leak any blocks beyond that dummy: freeing it + # must restore the exact pre-warmup free-block count. + kv_cache_manager.free_resources(padding_dummies[0]) self.assertEqual(num_free_before, kv_cache_manager.get_num_free_blocks()) kv_cache_manager.shutdown() + def test_warmup_skips_padding_dummy_when_padding_impossible(self): + # With max_batch_size=1 and a graph for batch size 1, every batch + # already matches a graph size, so no padded batch can ever occur and + # warmup must not retain a padding dummy (it would permanently hold + # KV blocks and spec/hybrid resources that are never used). + llm_args = TorchLlmArgs(model="dummy", + max_batch_size=1, + max_num_tokens=258, + cuda_graph_config=CudaGraphConfig( + enable_padding=True, batch_sizes=[1, 2, 4])) + model_engine, kv_cache_manager = create_model_engine_and_kvcache( + llm_args) + resource_manager = ResourceManager( + {ResourceManagerType.KV_CACHE_MANAGER: kv_cache_manager}) + + num_free_before = kv_cache_manager.get_num_free_blocks() + model_engine.warmup(resource_manager) + + self.assertEqual({}, + model_engine.cuda_graph_runner.padding_dummy_requests) + self.assertEqual(num_free_before, + kv_cache_manager.get_num_free_blocks()) + + kv_cache_manager.shutdown() + + def test_warmup_skips_padding_dummy_during_estimation(self): + # The estimation-phase KV cache is sized with no headroom for retained + # dummies; holding blocks there can leave the estimation requests + # unschedulable. Warmup must skip the preallocation for managers + # created for KV cache capacity estimation. + model_engine, kv_cache_manager = create_model_engine_and_kvcache() + kv_cache_manager.is_estimating_kv_cache = True + resource_manager = ResourceManager( + {ResourceManagerType.KV_CACHE_MANAGER: kv_cache_manager}) + + num_free_before = kv_cache_manager.get_num_free_blocks() + model_engine.warmup(resource_manager) + + self.assertEqual({}, + model_engine.cuda_graph_runner.padding_dummy_requests) + self.assertEqual(num_free_before, + kv_cache_manager.get_num_free_blocks()) + + kv_cache_manager.shutdown() + + def test_preallocate_padding_dummies_uses_captured_draft_lens(self): + # Speculative engines pad batches with the runtime draft length, so + # the preallocation must create dummies for the draft lengths of the + # captured graphs — not draft_len 0 unconditionally (an extra + # draft_len-0 dummy would permanently hold KV blocks that runtime + # padding never uses). + model_engine, kv_cache_manager = create_model_engine_and_kvcache() + resource_manager = ResourceManager( + {ResourceManagerType.KV_CACHE_MANAGER: kv_cache_manager}) + + runner = model_engine.cuda_graph_runner + # Simulate a spec-decode capture: graphs keyed (batch_size, draft_len, + # is_first_draft, short_seq_len_mode, is_all_greedy_sample) exist only + # for draft_len 2. + runner.graphs[(8, 2, False, False, True)] = Mock() + try: + runner.preallocate_padding_dummies(resource_manager) + + self.assertEqual([2], sorted(runner.padding_dummy_requests.keys())) + finally: + runner.graphs.clear() + for dummy in runner.padding_dummy_requests.values(): + kv_cache_manager.free_resources(dummy) + kv_cache_manager.shutdown() + def test_layerwise_nvtx_marker(self): llm_args = TorchLlmArgs( model="dummy", diff --git a/tests/unittest/_torch/speculative/test_eagle3.py b/tests/unittest/_torch/speculative/test_eagle3.py index 33a3667d0db4..b678377b2e6d 100644 --- a/tests/unittest/_torch/speculative/test_eagle3.py +++ b/tests/unittest/_torch/speculative/test_eagle3.py @@ -208,6 +208,39 @@ def test_eagle3_one_model_capture_uses_real_token_count() -> None: assert spec_metadata.hidden_states.shape == (4, 2) +def test_eagle3_resource_manager_shares_padding_dummy_slot() -> None: + """The target and draft engines of two-model EAGLE3 share one + Eagle3ResourceManager, and each registers its own CUDA graph padding dummy + under the same draft-length-derived request ID + (CUDA_GRAPH_DUMMY_REQUEST_ID - draft_len). The second registration must + reuse the already-reserved slot instead of tripping the strict re-add + assert in SlotManager.add_slot.""" + from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import \ + CUDA_GRAPH_DUMMY_REQUEST_ID + from tensorrt_llm._torch.speculative.eagle3 import Eagle3ResourceManager + + config = Eagle3DecodingConfig(max_draft_len=4, + speculative_model="/dummy/eagle3") + manager = Eagle3ResourceManager(config, + torch.half, + hidden_size=8, + max_num_requests=4, + max_seq_len=32, + max_num_tokens=64) + + dummy_request_id = CUDA_GRAPH_DUMMY_REQUEST_ID - config.max_draft_len + # The target engine registers the padding dummy first (e.g. during warmup + # preallocation), then the draft engine registers the same ID. + manager.add_dummy_requests([dummy_request_id]) + dummy_slot = manager.slot_manager.get_slot(dummy_request_id) + manager.add_dummy_requests([dummy_request_id]) + assert manager.slot_manager.get_slot(dummy_request_id) == dummy_slot + + # Real request IDs still get their own slots. + real_slot = manager.slot_manager.add_slot(7) + assert real_slot != dummy_slot + + @pytest.fixture(scope="function") def enforce_single_worker(monkeypatch): monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") @@ -1028,7 +1061,18 @@ def test_eagle3_cuda_graph_padding(disable_overlap_scheduler: bool): "The future of AI is", ] - sampling_params = SamplingParams(max_tokens=2048, temperature=0) + # Short generations keep all three requests within the KV cache budget so + # they run concurrently and batches of 3 actually pad up to the graph + # batch size 4. With long generations (2048 tokens against the 4096-token + # cache) the guaranteed-no-evict scheduler caps the batch at two requests, + # which exactly matches a graph batch size, so runtime padding (and the + # shared padding-dummy slot this test exists to cover) never engages. + # The overlap-scheduler variant must keep the long generations for now: + # padded draft batches trip a pre-existing shape mismatch in + # Eagle3SpecMetadata.prepare on the incremental-update path + # (hidden_states_read_indices sized without the padded requests). + max_tokens = 64 if disable_overlap_scheduler else 2048 + sampling_params = SamplingParams(max_tokens=max_tokens, temperature=0) llm_spec.generate(prompts, sampling_params) llm_spec.shutdown()