diff --git a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt index 8243c2cf03b5..1d98c4be4719 100644 --- a/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt +++ b/triton_backend/all_models/disaggregated_serving/disaggregated_serving_bls/config.pbtxt @@ -175,6 +175,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/gpt/ensemble/config.pbtxt b/triton_backend/all_models/gpt/ensemble/config.pbtxt index e1bc29a25311..a7377d1f4eda 100755 --- a/triton_backend/all_models/gpt/ensemble/config.pbtxt +++ b/triton_backend/all_models/gpt/ensemble/config.pbtxt @@ -58,6 +58,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -181,6 +187,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt b/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt index 5d2108fc3ea9..195b91e5b6b9 100644 --- a/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/gpt/tensorrt_llm/config.pbtxt @@ -71,6 +71,13 @@ input [ reshape: { shape: [ ] } optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + reshape: { shape: [ ] } + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt index c07a5ab6de44..8adeaea6661f 100644 --- a/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/ensemble/config.pbtxt @@ -104,6 +104,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -515,6 +521,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py index 9beb46d3bfdc..b4e867a1c91e 100755 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/1/model.py @@ -215,6 +215,8 @@ def get_sampling_config_from_request(request, batch_size=1, batch_index=0): request, 'frequency_penalty', batch_size, batch_index) kwargs['length_penalty'] = get_input_scalar_by_name(request, 'len_penalty', batch_size, batch_index) + kwargs['prompt_ignore_length'] = get_input_scalar_by_name( + request, 'prompt_ignore_length', batch_size, batch_index) kwargs['top_p_min'] = get_input_scalar_by_name(request, 'runtime_top_p_min', batch_size, batch_index) kwargs['top_p_reset_ids'] = get_input_scalar_by_name( diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt index cb3867183afc..9c808d69cb4c 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm/config.pbtxt @@ -205,6 +205,13 @@ input [ reshape: { shape: [ ] } optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + reshape: { shape: [ ] } + optional: true + }, { name: "early_stopping" data_type: TYPE_BOOL diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py index bf759b344bc5..66bb14299509 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/decode.py @@ -76,6 +76,7 @@ class Request: top_p: Optional[np.ndarray] = None temperature: Optional[np.ndarray] = None length_penalty: Optional[np.ndarray] = None + prompt_ignore_length: Optional[np.ndarray] = None repetition_penalty: Optional[np.ndarray] = None min_tokens: Optional[np.ndarray] = None return_log_probs: Optional[np.ndarray] = None diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py index 3383a1f8f503..55699b2559c2 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/1/lib/triton_decoder.py @@ -90,8 +90,8 @@ def __init__(self, "image_bytes_input", "image_url_input", "video_bytes_input", "max_tokens", "bad_words", "stop_words", "end_id", "pad_id", "top_k", "top_p", "temperature", "length_penalty", - "repetition_penalty", "min_tokens", "presence_penalty", - "frequency_penalty", "seed", "return_log_probs", + "prompt_ignore_length", "repetition_penalty", "min_tokens", + "presence_penalty", "frequency_penalty", "seed", "return_log_probs", "return_context_logits", "return_generation_logits", "beam_width", "stream", "prompt_embedding_table", "prompt_vocab_size", "prompt_table_extra_id", "embedding_bias_words", @@ -104,13 +104,13 @@ def __init__(self, self.__undo_reshape_whitelist = { "max_tokens", "end_id", "pad_id", "top_k", "top_p", "temperature", - "length_penalty", "repetition_penalty", "min_tokens", - "presence_penalty", "frequency_penalty", "seed", "return_log_probs", - "return_context_logits", "return_generation_logits", "beam_width", - "stream", "prompt_vocab_size", "num_draft_tokens", - "use_draft_logits", "exclude_input_in_output", - "return_perf_metrics", "lora_weights", "lora_config", - "lora_task_id", "return_num_input_tokens", + "length_penalty", "prompt_ignore_length", "repetition_penalty", + "min_tokens", "presence_penalty", "frequency_penalty", "seed", + "return_log_probs", "return_context_logits", + "return_generation_logits", "beam_width", "stream", + "prompt_vocab_size", "num_draft_tokens", "use_draft_logits", + "exclude_input_in_output", "return_perf_metrics", "lora_weights", + "lora_config", "lora_task_id", "return_num_input_tokens", "return_num_output_tokens" } @@ -455,6 +455,7 @@ def _get_llm_tensors_from_request( "top_p": "runtime_top_p", "temperature": "temperature", "length_penalty": "len_penalty", + "prompt_ignore_length": "prompt_ignore_length", "repetition_penalty": "repetition_penalty", "min_tokens": "min_tokens", "presence_penalty": "presence_penalty", diff --git a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt index 7ad5ccf9f18c..8dbb66a7161c 100644 --- a/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt +++ b/triton_backend/all_models/inflight_batcher_llm/tensorrt_llm_bls/config.pbtxt @@ -134,6 +134,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 diff --git a/triton_backend/all_models/multimodal/ensemble/config.pbtxt b/triton_backend/all_models/multimodal/ensemble/config.pbtxt index 777118eefe66..6a0d41e6babd 100755 --- a/triton_backend/all_models/multimodal/ensemble/config.pbtxt +++ b/triton_backend/all_models/multimodal/ensemble/config.pbtxt @@ -123,6 +123,12 @@ input [ dims: [ 1 ] optional: true }, + { + name: "prompt_ignore_length" + data_type: TYPE_INT32 + dims: [ 1 ] + optional: true + }, { name: "repetition_penalty" data_type: TYPE_FP32 @@ -512,6 +518,10 @@ ensemble_scheduling { key: "len_penalty" value: "length_penalty" } + input_map { + key: "prompt_ignore_length" + value: "prompt_ignore_length" + } input_map { key: "repetition_penalty" value: "repetition_penalty" diff --git a/triton_backend/inflight_batcher_llm/src/utils.cc b/triton_backend/inflight_batcher_llm/src/utils.cc index 5ccc586e7856..f714649b606f 100644 --- a/triton_backend/inflight_batcher_llm/src/utils.cc +++ b/triton_backend/inflight_batcher_llm/src/utils.cc @@ -595,6 +595,9 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs std::optional frequencyPenalty{std::nullopt}; extractOptionalSingleton(inputsTensors, InputFieldsNames::frequencyPenalty, frequencyPenalty); + std::optional promptIgnoreLength{std::nullopt}; + extractOptionalSingleton(inputsTensors, InputFieldsNames::promptIgnoreLength, promptIgnoreLength); + std::optional seed{std::nullopt}; extractOptionalSingleton(inputsTensors, InputFieldsNames::seed, seed); @@ -605,8 +608,8 @@ executor::SamplingConfig getSamplingConfigFromTensors(InputTensors const& inputs extractOptionalSingleton(inputsTensors, InputFieldsNames::numReturnSequences, numReturnSequences); return executor::SamplingConfig(beamWidth, topK, topP, topPMin, topPResetIds, topPDecay, seed, temperature, - minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, lengthPenalty, - earlyStopping, noRepeatNgramSize, numReturnSequences); + minTokens, beamSearchDiversityRate, repetitionPenalty, presencePenalty, frequencyPenalty, promptIgnoreLength, + lengthPenalty, earlyStopping, noRepeatNgramSize, numReturnSequences); } executor::OutputConfig getOutputConfigFromTensors(InputTensors const& inputsTensors) diff --git a/triton_backend/inflight_batcher_llm/src/utils.h b/triton_backend/inflight_batcher_llm/src/utils.h index 78e0ec6545c2..6249bfdfa714 100644 --- a/triton_backend/inflight_batcher_llm/src/utils.h +++ b/triton_backend/inflight_batcher_llm/src/utils.h @@ -96,6 +96,7 @@ struct InputFieldsNames static constexpr char const* beamSearchDiversityRate = "beam_search_diversity_rate"; static constexpr char const* presencePenalty = "presence_penalty"; static constexpr char const* frequencyPenalty = "frequency_penalty"; + static constexpr char const* promptIgnoreLength = "prompt_ignore_length"; static constexpr char const* seed = "seed"; // PromptTuningConfig diff --git a/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp b/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp index cd6a5c78048c..fa0b84ccdcbc 100644 --- a/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp +++ b/triton_backend/inflight_batcher_llm/tests/utilsTest.cpp @@ -388,6 +388,7 @@ std::optional getRequest( pushTensor(inputsTensors, InputFieldsNames::beamSearchDiversityRate, nvinfer1::DataType::kFLOAT, {1}, {0.1}); pushTensor(inputsTensors, InputFieldsNames::presencePenalty, nvinfer1::DataType::kFLOAT, {1}, {0.2}); pushTensor(inputsTensors, InputFieldsNames::frequencyPenalty, nvinfer1::DataType::kFLOAT, {1}, {0.3}); + pushTensor(inputsTensors, InputFieldsNames::promptIgnoreLength, nvinfer1::DataType::kINT32, {1}, {7}); pushTensor(inputsTensors, InputFieldsNames::seed, nvinfer1::DataType::kINT64, {1}, {3456}); // PromptTuningConfig @@ -585,6 +586,7 @@ void checkRequest(tensorrt_llm::executor::Request const& request, EXPECT_EQ(samplingConfig.getBeamSearchDiversityRate().value(), 0.1f); EXPECT_EQ(samplingConfig.getPresencePenalty().value(), 0.2f); EXPECT_EQ(samplingConfig.getFrequencyPenalty().value(), 0.3f); + EXPECT_EQ(samplingConfig.getPromptIgnoreLength().value(), 7); EXPECT_EQ(samplingConfig.getSeed().value(), 3456); EXPECT_EQ(samplingConfig.getNumReturnSequences().value(), 29);