diff --git a/cpp/tensorrt_llm/nanobind/executor/request.cpp b/cpp/tensorrt_llm/nanobind/executor/request.cpp index 97b77e2214a2..510bd2fd343b 100644 --- a/cpp/tensorrt_llm/nanobind/executor/request.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/request.cpp @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -591,7 +592,15 @@ void initRequestBindings(nb::module_& m) auto requestGetstate = [](tle::Request const& self) { - return nb::make_tuple(self.getInputTokenIds(), self.getMaxTokens(), self.getStreaming(), + // Serialize input_token_ids as a raw int32 byte buffer instead of a Python + // list[int]: nanobind casts VecTokens element-by-element (a PyLong storm, + // ISL-proportional) on every Request pickle -- the request broadcast and the + // RPC IPC submit. A bytes blob is a memcpy, ~order-of-magnitude cheaper. + // Paired with requestSetstate. + auto const& inputTokenIds = self.getInputTokenIds(); + auto inputTokenIdsBytes = nb::bytes( + reinterpret_cast(inputTokenIds.data()), inputTokenIds.size() * sizeof(VecTokens::value_type)); + return nb::make_tuple(std::move(inputTokenIdsBytes), self.getMaxTokens(), self.getStreaming(), self.getSamplingConfig(), self.getOutputConfig(), self.getEndId(), self.getPadId(), self.getPositionIds(), self.getBadWords(), self.getStopWords(), self.getEmbeddingBias(), self.getExternalDraftTokensConfig(), self.getPromptTuningConfig(), self.getMultimodalInput(), self.getMultimodalEmbedding(), @@ -608,8 +617,12 @@ void initRequestBindings(nb::module_& m) { throw std::runtime_error("Invalid Request state!"); } - new (&self) tle::Request(nb::cast(state[0]), nb::cast(state[1]), - nb::cast(state[2]), nb::cast(state[3]), nb::cast(state[4]), + // input_token_ids is a raw int32 byte buffer (see requestGetstate). + auto const inputTokenIdsBytes = nb::cast(state[0]); + VecTokens inputTokenIds(inputTokenIdsBytes.size() / sizeof(VecTokens::value_type)); + std::memcpy(inputTokenIds.data(), inputTokenIdsBytes.c_str(), inputTokenIdsBytes.size()); + new (&self) tle::Request(std::move(inputTokenIds), nb::cast(state[1]), nb::cast(state[2]), + nb::cast(state[3]), nb::cast(state[4]), nb::cast>(state[5]), nb::cast>(state[6]), nb::cast>>(state[7]), nb::cast>>(state[8]),