[None][fix] Plumb promptIgnoreLength through Triton backend to fix silently-dropped lengthPenalty and earlyStopping

triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt

@@ -175,6 +175,12 @@ input [
     dims: [ 1 ]
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32

triton_backend/all_models/gpt/ensemble/config.pbtxt

@@ -58,6 +58,12 @@ input [
     dims: [ 1 ]
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32
@@ -181,6 +187,10 @@ ensemble_scheduling {
           key: "len_penalty"
           value: "length_penalty"
       }
+      input_map {
+          key: "prompt_ignore_length"
+          value: "prompt_ignore_length"
+      }
       input_map {
           key: "repetition_penalty"
           value: "repetition_penalty"

triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt

@@ -71,6 +71,13 @@ input [
     reshape: { shape: [ ] }
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    reshape: { shape: [ ] }
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32

triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt

@@ -104,6 +104,12 @@ input [
     dims: [ 1 ]
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32
@@ -515,6 +521,10 @@ ensemble_scheduling {
           key: "len_penalty"
           value: "length_penalty"
       }
+      input_map {
+          key: "prompt_ignore_length"
+          value: "prompt_ignore_length"
+      }
       input_map {
           key: "repetition_penalty"
           value: "repetition_penalty"

triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py

@@ -215,6 +215,8 @@ def get_sampling_config_from_request(request, batch_size=1, batch_index=0):
         request, 'frequency_penalty', batch_size, batch_index)
     kwargs['length_penalty'] = get_input_scalar_by_name(request, 'len_penalty',
                                                         batch_size, batch_index)
+    kwargs['prompt_ignore_length'] = get_input_scalar_by_name(
+        request, 'prompt_ignore_length', batch_size, batch_index)
     kwargs['top_p_min'] = get_input_scalar_by_name(request, 'runtime_top_p_min',
                                                    batch_size, batch_index)
     kwargs['top_p_reset_ids'] = get_input_scalar_by_name(

triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt

@@ -205,6 +205,13 @@ input [
     reshape: { shape: [ ] }
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    reshape: { shape: [ ] }
+    optional: true
+  },
   {
     name: "early_stopping"
     data_type: TYPE_BOOL

triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py

@@ -76,6 +76,7 @@ class Request:
     top_p: Optional[np.ndarray] = None
     temperature: Optional[np.ndarray] = None
     length_penalty: Optional[np.ndarray] = None
+    prompt_ignore_length: Optional[np.ndarray] = None
     repetition_penalty: Optional[np.ndarray] = None
     min_tokens: Optional[np.ndarray] = None
     return_log_probs: Optional[np.ndarray] = None

triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py

@@ -90,8 +90,8 @@ def __init__(self,
             "image_bytes_input", "image_url_input", "video_bytes_input",
             "max_tokens", "bad_words", "stop_words", "end_id", "pad_id",
             "top_k", "top_p", "temperature", "length_penalty",
-            "repetition_penalty", "min_tokens", "presence_penalty",
-            "frequency_penalty", "seed", "return_log_probs",
+            "prompt_ignore_length", "repetition_penalty", "min_tokens",
+            "presence_penalty", "frequency_penalty", "seed", "return_log_probs",
             "return_context_logits", "return_generation_logits", "beam_width",
             "stream", "prompt_embedding_table", "prompt_vocab_size",
             "prompt_table_extra_id", "embedding_bias_words",
@@ -104,13 +104,13 @@ def __init__(self,
 
         self.__undo_reshape_whitelist = {
             "max_tokens", "end_id", "pad_id", "top_k", "top_p", "temperature",
-            "length_penalty", "repetition_penalty", "min_tokens",
-            "presence_penalty", "frequency_penalty", "seed", "return_log_probs",
-            "return_context_logits", "return_generation_logits", "beam_width",
-            "stream", "prompt_vocab_size", "num_draft_tokens",
-            "use_draft_logits", "exclude_input_in_output",
-            "return_perf_metrics", "lora_weights", "lora_config",
-            "lora_task_id", "return_num_input_tokens",
+            "length_penalty", "prompt_ignore_length", "repetition_penalty",
+            "min_tokens", "presence_penalty", "frequency_penalty", "seed",
+            "return_log_probs", "return_context_logits",
+            "return_generation_logits", "beam_width", "stream",
+            "prompt_vocab_size", "num_draft_tokens", "use_draft_logits",
+            "exclude_input_in_output", "return_perf_metrics", "lora_weights",
+            "lora_config", "lora_task_id", "return_num_input_tokens",
             "return_num_output_tokens"
         }
 
@@ -455,6 +455,7 @@ def _get_llm_tensors_from_request(
             "top_p": "runtime_top_p",
             "temperature": "temperature",
             "length_penalty": "len_penalty",
+            "prompt_ignore_length": "prompt_ignore_length",
             "repetition_penalty": "repetition_penalty",
             "min_tokens": "min_tokens",
             "presence_penalty": "presence_penalty",

triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt

@@ -134,6 +134,12 @@ input [
     dims: [ 1 ]
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32

triton_backend/all_models/multimodal/ensemble/config.pbtxt

@@ -123,6 +123,12 @@ input [
     dims: [ 1 ]
     optional: true
   },
+  {
+    name: "prompt_ignore_length"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
   {
     name: "repetition_penalty"
     data_type: TYPE_FP32
@@ -512,6 +518,10 @@ ensemble_scheduling {
           key: "len_penalty"
           value: "length_penalty"
       }
+      input_map {
+          key: "prompt_ignore_length"
+          value: "prompt_ignore_length"
+      }
       input_map {
           key: "repetition_penalty"
           value: "repetition_penalty"

triton_backend/inflight_batcher_llm/src/utils.cc

@@ -595,6 +595,9 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs
     std::optional<float> frequencyPenalty{std::nullopt};
     extractOptionalSingleton<float>(inputsTensors, InputFieldsNames::frequencyPenalty, frequencyPenalty);
 
+    std::optional<int32_t> promptIgnoreLength{std::nullopt};
+    extractOptionalSingleton<int32_t>(inputsTensors, InputFieldsNames::promptIgnoreLength, promptIgnoreLength);
+
     std::optional<uint64_t> seed{std::nullopt};
     extractOptionalSingleton<uint64_t>(inputsTensors, InputFieldsNames::seed, seed);
 
@@ -605,8 +608,8 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs
     extractOptionalSingleton<int32_t>(inputsTensors, InputFieldsNames::numReturnSequences, numReturnSequences);
 
     return executor::SamplingConfig(beamWidth, topK, topP, topPMin, topPResetIds, topPDecay, seed, temperature,
-        minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, lengthPenalty,
-        earlyStopping, noRepeatNgramSize, numReturnSequences);
+        minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, promptIgnoreLength,
+        lengthPenalty, earlyStopping, noRepeatNgramSize, numReturnSequences);
 }
 
 executor::OutputConfig getOutputConfigFromTensors(InputTensors const& inputsTensors)

triton_backend/inflight_batcher_llm/src/utils.h

@@ -96,6 +96,7 @@ struct InputFieldsNames
     static constexpr char const* beamSearchDiversityRate = "beam_search_diversity_rate";
     static constexpr char const* presencePenalty = "presence_penalty";
     static constexpr char const* frequencyPenalty = "frequency_penalty";
+    static constexpr char const* promptIgnoreLength = "prompt_ignore_length";
     static constexpr char const* seed = "seed";
 
     // PromptTuningConfig

triton_backend/inflight_batcher_llm/tests/utilsTest.cpp

@@ -388,6 +388,7 @@ std::optional<tensorrt_llm::executor::Request> getRequest(
     pushTensor<float>(inputsTensors, InputFieldsNames::beamSearchDiversityRate, nvinfer1::DataType::kFLOAT, {1}, {0.1});
     pushTensor<float>(inputsTensors, InputFieldsNames::presencePenalty, nvinfer1::DataType::kFLOAT, {1}, {0.2});
     pushTensor<float>(inputsTensors, InputFieldsNames::frequencyPenalty, nvinfer1::DataType::kFLOAT, {1}, {0.3});
+    pushTensor<int32_t>(inputsTensors, InputFieldsNames::promptIgnoreLength, nvinfer1::DataType::kINT32, {1}, {7});
     pushTensor<uint64_t>(inputsTensors, InputFieldsNames::seed, nvinfer1::DataType::kINT64, {1}, {3456});
 
     // PromptTuningConfig
@@ -585,6 +586,7 @@ void checkRequest(tensorrt_llm::executor::Request const& request,
     EXPECT_EQ(samplingConfig.getBeamSearchDiversityRate().value(), 0.1f);
     EXPECT_EQ(samplingConfig.getPresencePenalty().value(), 0.2f);
     EXPECT_EQ(samplingConfig.getFrequencyPenalty().value(), 0.3f);
+    EXPECT_EQ(samplingConfig.getPromptIgnoreLength().value(), 7);
     EXPECT_EQ(samplingConfig.getSeed().value(), 3456);
     EXPECT_EQ(samplingConfig.getNumReturnSequences().value(), 29);