From bafcf33a32016f1299078c4a3b98cb9ad8653b37 Mon Sep 17 00:00:00 2001 From: Jhao-Ting Chen Date: Wed, 29 Apr 2026 17:57:26 -0700 Subject: [PATCH] fix: Plumb promptIgnoreLength through Triton backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor::SamplingConfig constructor takes 20 positional std::optional parameters, with promptIgnoreLength (added in PR #8127) at position 14. The Triton backend's getSamplingConfigFromTensors() in utils.cc was only passing 17 args, omitting promptIgnoreLength. This compiled silently because std::optional implicitly converts to std::optional via the contained type, so all params from #14 onward shifted positions: caller's lengthPenalty bound to promptIgnoreLength, earlyStopping bound to lengthPenalty, etc. As a result, length_penalty and early_stopping sent over Triton (gRPC/HTTP) were silently ignored, and prompt_ignore_length had no way to be set at all. This change adds full plumbing for prompt_ignore_length so callers can configure it from the Triton client all the way through to the executor and the penalty kernels: - triton_backend/inflight_batcher_llm/src/utils.{h,cc}: declare new InputFieldsNames::promptIgnoreLength input field, extract it from input tensors via extractOptionalSingleton, and pass it into executor::SamplingConfig at position 14 (replacing the silent default). - triton_backend/inflight_batcher_llm/tests/utilsTest.cpp: extend the extractSingleton fixture to push a prompt_ignore_length tensor and assert SamplingConfig::getPromptIgnoreLength() round-trips correctly. - Triton model configs in all_models/ — declare optional INT32 prompt_ignore_length input on every model that already exposes len_penalty (the sibling sampling field), and add the corresponding ensemble input_map entry where applicable: inflight_batcher_llm/{tensorrt_llm,tensorrt_llm_bls,ensemble}/config.pbtxt disaggregated_serving/disaggregated_serving_bls/config.pbtxt gpt/{tensorrt_llm,ensemble}/config.pbtxt multimodal/ensemble/config.pbtxt - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py: forward prompt_ignore_length from request to trtllm.SamplingConfig kwargs (covers both Triton+engine and Triton+LLMAPI flows). - triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/{decode,triton_decoder}.py: add prompt_ignore_length to the BLS Request dataclass, the input list, and the BLS->engine name mapping. Backward compatibility: prompt_ignore_length is an optional input. When the tensor is not provided, getSamplingConfigFromTensors yields std::nullopt, matching the previous (already broken) default behavior, but now also correctly forwarding lengthPenalty/earlyStopping/etc to their intended SamplingConfig slots. Signed-off-by: Jhao-Ting Chen --- .../disaggregated_serving_bls/config.pbtxt | 6 ++++++ .../all_models/gpt/ensemble/config.pbtxt | 10 ++++++++++ .../all_models/gpt/tensorrt_llm/config.pbtxt | 7 +++++++ .../ensemble/config.pbtxt | 10 ++++++++++ .../tensorrt_llm/1/model.py | 2 ++ .../tensorrt_llm/config.pbtxt | 7 +++++++ .../tensorrt_llm_bls/1/lib/decode.py | 1 + .../tensorrt_llm_bls/1/lib/triton_decoder.py | 19 ++++++++++--------- .../tensorrt_llm_bls/config.pbtxt | 6 ++++++ .../multimodal/ensemble/config.pbtxt | 10 ++++++++++ .../inflight_batcher_llm/src/utils.cc | 7 +++++-- .../inflight_batcher_llm/src/utils.h | 1 + .../inflight_batcher_llm/tests/utilsTest.cpp | 2 ++ 13 files changed, 77 insertions(+), 11 deletions(-) diff --git a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt index 8243c2cf03b5..1d98c4be4719 100644 --- a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt +++ b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt @@ -175,6 +175,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/gpt/ensemble/config.pbtxt b/triton_backend/all_models/gpt/ensemble/config.pbtxt index e1bc29a25311..a7377d1f4eda 100755 --- a/triton_backend/all_models/gpt/ensemble/config.pbtxt +++ b/triton_backend/all_models/gpt/ensemble/config.pbtxt @@ -58,6 +58,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -181,6 +187,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt b/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt index 5d2108fc3ea9..195b91e5b6b9 100644 --- a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt @@ -71,6 +71,13 @@ input [ reshape: { shape: [ ] } optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + reshape: { shape: [ ] } + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt index c07a5ab6de44..8adeaea6661f 100644 --- a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt @@ -104,6 +104,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -515,6 +521,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py index 9beb46d3bfdc..b4e867a1c91e 100755 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py @@ -215,6 +215,8 @@ def get_sampling_config_from_request(request, batch_size=1, batch_index=0): request, 'frequency_penalty', batch_size, batch_index) kwargs['length_penalty'] = get_input_scalar_by_name(request, 'len_penalty', batch_size, batch_index) + kwargs['prompt_ignore_length'] = get_input_scalar_by_name( + request, 'prompt_ignore_length', batch_size, batch_index) kwargs['top_p_min'] = get_input_scalar_by_name(request, 'runtime_top_p_min', batch_size, batch_index) kwargs['top_p_reset_ids'] = get_input_scalar_by_name( diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt index cb3867183afc..9c808d69cb4c 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt @@ -205,6 +205,13 @@ input [ reshape: { shape: [ ] } optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + reshape: { shape: [ ] } + optional: true + }, { name: "early_stopping" data_type: TYPE_BOOL diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py index bf759b344bc5..66bb14299509 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py @@ -76,6 +76,7 @@ class Request: top_p: Optional[np.ndarray] = None temperature: Optional[np.ndarray] = None length_penalty: Optional[np.ndarray] = None + prompt_ignore_length: Optional[np.ndarray] = None repetition_penalty: Optional[np.ndarray] = None min_tokens: Optional[np.ndarray] = None return_log_probs: Optional[np.ndarray] = None diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py index 3383a1f8f503..55699b2559c2 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py @@ -90,8 +90,8 @@ def __init__(self, "image_bytes_input", "image_url_input", "video_bytes_input", "max_tokens", "bad_words", "stop_words", "end_id", "pad_id", "top_k", "top_p", "temperature", "length_penalty", - "repetition_penalty", "min_tokens", "presence_penalty", - "frequency_penalty", "seed", "return_log_probs", + "prompt_ignore_length", "repetition_penalty", "min_tokens", + "presence_penalty", "frequency_penalty", "seed", "return_log_probs", "return_context_logits", "return_generation_logits", "beam_width", "stream", "prompt_embedding_table", "prompt_vocab_size", "prompt_table_extra_id", "embedding_bias_words", @@ -104,13 +104,13 @@ def __init__(self, self.__undo_reshape_whitelist = { "max_tokens", "end_id", "pad_id", "top_k", "top_p", "temperature", - "length_penalty", "repetition_penalty", "min_tokens", - "presence_penalty", "frequency_penalty", "seed", "return_log_probs", - "return_context_logits", "return_generation_logits", "beam_width", - "stream", "prompt_vocab_size", "num_draft_tokens", - "use_draft_logits", "exclude_input_in_output", - "return_perf_metrics", "lora_weights", "lora_config", - "lora_task_id", "return_num_input_tokens", + "length_penalty", "prompt_ignore_length", "repetition_penalty", + "min_tokens", "presence_penalty", "frequency_penalty", "seed", + "return_log_probs", "return_context_logits", + "return_generation_logits", "beam_width", "stream", + "prompt_vocab_size", "num_draft_tokens", "use_draft_logits", + "exclude_input_in_output", "return_perf_metrics", "lora_weights", + "lora_config", "lora_task_id", "return_num_input_tokens", "return_num_output_tokens" } @@ -455,6 +455,7 @@ def _get_llm_tensors_from_request( "top_p": "runtime_top_p", "temperature": "temperature", "length_penalty": "len_penalty", + "prompt_ignore_length": "prompt_ignore_length", "repetition_penalty": "repetition_penalty", "min_tokens": "min_tokens", "presence_penalty": "presence_penalty", diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt index 7ad5ccf9f18c..8dbb66a7161c 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt @@ -134,6 +134,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/multimodal/ensemble/config.pbtxt b/triton_backend/all_models/multimodal/ensemble/config.pbtxt index 777118eefe66..6a0d41e6babd 100755 --- a/triton_backend/all_models/multimodal/ensemble/config.pbtxt +++ b/triton_backend/all_models/multimodal/ensemble/config.pbtxt @@ -123,6 +123,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -512,6 +518,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/inflight_batcher_llm/src/utils.cc b/triton_backend/inflight_batcher_llm/src/utils.cc index 5ccc586e7856..f714649b606f 100644 --- a/triton_backend/inflight_batcher_llm/src/utils.cc +++ b/triton_backend/inflight_batcher_llm/src/utils.cc @@ -595,6 +595,9 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs std::optional frequencyPenalty{std::nullopt}; extractOptionalSingleton(inputsTensors, InputFieldsNames::frequencyPenalty, frequencyPenalty); + std::optional promptIgnoreLength{std::nullopt}; + extractOptionalSingleton(inputsTensors, InputFieldsNames::promptIgnoreLength, promptIgnoreLength); + std::optional seed{std::nullopt}; extractOptionalSingleton(inputsTensors, InputFieldsNames::seed, seed); @@ -605,8 +608,8 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs extractOptionalSingleton(inputsTensors, InputFieldsNames::numReturnSequences, numReturnSequences); return executor::SamplingConfig(beamWidth, topK, topP, topPMin, topPResetIds, topPDecay, seed, temperature, - minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, lengthPenalty, - earlyStopping, noRepeatNgramSize, numReturnSequences); + minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, promptIgnoreLength, + lengthPenalty, earlyStopping, noRepeatNgramSize, numReturnSequences); } executor::OutputConfig getOutputConfigFromTensors(InputTensors const& inputsTensors) diff --git a/triton_backend/inflight_batcher_llm/src/utils.h b/triton_backend/inflight_batcher_llm/src/utils.h index 78e0ec6545c2..6249bfdfa714 100644 --- a/triton_backend/inflight_batcher_llm/src/utils.h +++ b/triton_backend/inflight_batcher_llm/src/utils.h @@ -96,6 +96,7 @@ struct InputFieldsNames static constexpr char const* beamSearchDiversityRate = "beam_search_diversity_rate"; static constexpr char const* presencePenalty = "presence_penalty"; static constexpr char const* frequencyPenalty = "frequency_penalty"; + static constexpr char const* promptIgnoreLength = "prompt_ignore_length"; static constexpr char const* seed = "seed"; // PromptTuningConfig diff --git a/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp b/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp index cd6a5c78048c..fa0b84ccdcbc 100644 --- a/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp +++ b/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp @@ -388,6 +388,7 @@ std::optional getRequest( pushTensor(inputsTensors, InputFieldsNames::beamSearchDiversityRate, nvinfer1::DataType::kFLOAT, {1}, {0.1}); pushTensor(inputsTensors, InputFieldsNames::presencePenalty, nvinfer1::DataType::kFLOAT, {1}, {0.2}); pushTensor(inputsTensors, InputFieldsNames::frequencyPenalty, nvinfer1::DataType::kFLOAT, {1}, {0.3}); + pushTensor(inputsTensors, InputFieldsNames::promptIgnoreLength, nvinfer1::DataType::kINT32, {1}, {7}); pushTensor(inputsTensors, InputFieldsNames::seed, nvinfer1::DataType::kINT64, {1}, {3456}); // PromptTuningConfig @@ -585,6 +586,7 @@ void checkRequest(tensorrt_llm::executor::Request const& request, EXPECT_EQ(samplingConfig.getBeamSearchDiversityRate().value(), 0.1f); EXPECT_EQ(samplingConfig.getPresencePenalty().value(), 0.2f); EXPECT_EQ(samplingConfig.getFrequencyPenalty().value(), 0.3f); + EXPECT_EQ(samplingConfig.getPromptIgnoreLength().value(), 7); EXPECT_EQ(samplingConfig.getSeed().value(), 3456); EXPECT_EQ(samplingConfig.getNumReturnSequences().value(), 29);