[Feat][Deploy] Add ultrainfer and paddlex-hpi (PaddlePaddle#2625)

Author: zhang-prog

.precommit/check_custom.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 import sys
 
@@ -40,9 +41,12 @@ def check(file_path):
     if not content.startswith(LICENSE_TEXT):
         print(f"License header missing in {file_path}")
         return False
-    if "import paddle" in content or "from paddle import " in content:
-        print(f"Please use `lazy_paddle` instead `paddle` when import in {file_path}")
-        return False
+    if "paddlex" in file_path.split(os.sep):
+        if "import paddle" in content or "from paddle import " in content:
+            print(
+                f"Please use `lazy_paddle` instead `paddle` when import in {file_path}"
+            )
+            return False
     return True
 
 

libs/paddlex-hpi/MANIFEST.in

@@ -0,0 +1,2 @@
+include src/paddlex_hpi/py.typed
+include src/paddlex_hpi/model_info_collection.json

libs/paddlex-hpi/README.md

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools >= 69"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "paddlex-hpi"
+version = "3.0.0.b2"
+description = ""
+readme = "README.md"
+authors = []
+dynamic = ["dependencies", "optional-dependencies"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies.test = {file = ["test_requirements.txt"]}

libs/paddlex-hpi/pyproject.toml

@@ -0,0 +1,7 @@
+# ultrainfer
+# paddlex
+importlib-resources >= 6.4
+numpy >= 1.21
+pandas >= 1.3.3
+pydantic >= 2
+typing-extensions >= 4.11

libs/paddlex-hpi/requirements.txt

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python -m pip wheel -w wheels/original --no-deps .

libs/paddlex-hpi/scripts/build_wheel.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python -m pytest tests

libs/paddlex-hpi/scripts/run_tests.sh

@@ -0,0 +1,15 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "3.0.0.beta2"

libs/paddlex-hpi/src/paddlex_hpi/__init__.py

@@ -0,0 +1,218 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from pathlib import Path
+from typing import Any, Dict, List, Mapping, Optional, Tuple, Type, Union
+
+import ultrainfer as ui
+from paddlex.utils import logging
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from typing_extensions import Annotated, TypeAlias, TypedDict, assert_never
+
+from paddlex_hpi._model_info import get_model_info
+from paddlex_hpi._utils.typing import Backend, DeviceType
+
+
+class _BackendConfig(BaseModel):
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        raise NotImplementedError
+
+
+class PaddleInferConfig(_BackendConfig):
+    cpu_num_threads: int = 8
+    enable_mkldnn: bool = True
+    enable_trt: bool = False
+    trt_dynamic_shapes: Optional[Dict[str, List[List[int]]]] = None
+    trt_dynamic_shape_input_data: Optional[Dict[str, List[List[float]]]] = None
+    enable_log_info: bool = False
+
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        option.use_paddle_infer_backend()
+        option.set_cpu_thread_num(self.cpu_num_threads)
+        option.paddle_infer_option.enable_mkldnn = self.enable_mkldnn
+        option.paddle_infer_option.enable_trt = self.enable_trt
+        option.trt_option.serialize_file = str(model_dir / "trt_serialized.trt")
+        if self.trt_dynamic_shapes is not None:
+            for name, shapes in self.trt_dynamic_shapes.items():
+                option.trt_option.set_shape(name, *shapes)
+        if self.trt_dynamic_shape_input_data is not None:
+            for name, data in self.trt_dynamic_shape_input_data.items():
+                option.trt_option.set_input_data(name, *data)
+        if self.enable_trt:
+            option.paddle_infer_option.collect_trt_shape = True
+            option.paddle_infer_option.collect_trt_shape_by_device = True
+        option.paddle_infer_option.enable_log_info = self.enable_log_info
+
+
+class OpenVINOConfig(_BackendConfig):
+    cpu_num_threads: int = 8
+
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        option.use_openvino_backend()
+        option.set_cpu_thread_num(self.cpu_num_threads)
+
+
+class ONNXRuntimeConfig(_BackendConfig):
+    cpu_num_threads: int = 8
+
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        option.use_ort_backend()
+        option.set_cpu_thread_num(self.cpu_num_threads)
+
+
+class TensorRTConfig(_BackendConfig):
+    dynamic_shapes: Optional[Dict[str, List[List[int]]]] = None
+
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        option.use_trt_backend()
+        option.trt_option.serialize_file = str(model_dir / "trt_serialized.trt")
+        if self.dynamic_shapes is not None:
+            for name, shapes in self.dynamic_shapes.items():
+                option.trt_option.set_shape(name, *shapes)
+
+
+class PaddleTensorRTConfig(_BackendConfig):
+    dynamic_shapes: Dict[str, List[List[int]]]
+    dynamic_shape_input_data: Optional[Dict[str, List[List[float]]]] = None
+    enable_log_info: bool = False
+
+    def update_ui_option(self, option: ui.RuntimeOption, model_dir: Path) -> None:
+        option.use_paddle_infer_backend()
+        option.paddle_infer_option.enable_trt = True
+        option.trt_option.serialize_file = str(model_dir / "trt_serialized.trt")
+        if self.dynamic_shapes is not None:
+            option.paddle_infer_option.collect_trt_shape = True
+            # TODO: Support setting collect_trt_shape_by_device
+            for name, shapes in self.dynamic_shapes.items():
+                option.trt_option.set_shape(name, *shapes)
+        if self.dynamic_shape_input_data is not None:
+            for name, data in self.dynamic_shape_input_data.items():
+                option.trt_option.set_input_data(name, *data)
+        option.paddle_infer_option.enable_log_info = self.enable_log_info
+
+
+# Should we use tagged unions?
+BackendConfig: TypeAlias = Union[
+    PaddleInferConfig,
+    OpenVINOConfig,
+    ONNXRuntimeConfig,
+    TensorRTConfig,
+]
+
+
+def get_backend_config_type(backend: Backend, /) -> Type[BackendConfig]:
+    backend_config_type: Type[BackendConfig]
+    if backend == "paddle_infer":
+        backend_config_type = PaddleInferConfig
+    elif backend == "openvino":
+        backend_config_type = OpenVINOConfig
+    elif backend == "onnx_runtime":
+        backend_config_type = ONNXRuntimeConfig
+    elif backend == "tensorrt":
+        backend_config_type = TensorRTConfig
+    else:
+        assert_never(backend)
+    return backend_config_type
+
+
+# Can I create this dynamically and automatically?
+class BackendConfigs(TypedDict, total=False):
+    paddle_infer: PaddleInferConfig
+    openvino: OpenVINOConfig
+    onnx_runtime: ONNXRuntimeConfig
+    tensorrt: TensorRTConfig
+    paddle_tensorrt: PaddleTensorRTConfig
+
+
+class HPIConfig(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    selected_backends: Optional[Dict[DeviceType, Backend]] = None
+    # For backward compatilibity
+    backend_configs: Annotated[
+        Optional[BackendConfigs], Field(validation_alias="backend_config")
+    ] = None
+
+    def get_backend_and_config(
+        self, model_name: str, device_type: DeviceType
+    ) -> Tuple[Backend, BackendConfig]:
+        # Do we need an extensible selector?
+        model_info = get_model_info(model_name, device_type)
+        if model_info:
+            backend_config_pairs = model_info["backend_config_pairs"]
+        else:
+            backend_config_pairs = []
+        config_dict: Dict[str, Any] = {}
+        if self.selected_backends and device_type in self.selected_backends:
+            backend = self.selected_backends[device_type]
+            for pair in backend_config_pairs:
+                # Use the first one
+                if pair[0] == self.selected_backends[device_type]:
+                    config_dict.update(pair[1])
+                    break
+        else:
+            if backend_config_pairs:
+                # Currently we select the first one
+                backend = backend_config_pairs[0][0]
+                config_dict.update(backend_config_pairs[0][1])
+            else:
+                backend = "paddle_infer"
+        if self.backend_configs and backend in self.backend_configs:
+            config_dict.update(
+                self.backend_configs[backend].model_dump(exclude_unset=True)
+            )
+        backend_config_type = get_backend_config_type(backend)
+        backend_config = backend_config_type.model_validate(config_dict)
+        return backend, backend_config
+
+    # XXX: For backward compatilibity
+    @field_validator("selected_backends", mode="before")
+    @classmethod
+    def _hack_selected_backends(cls, data: Any) -> Any:
+        if isinstance(data, Mapping):
+            new_data = dict(data)
+            for device_type in new_data:
+                if new_data[device_type] == "paddle_tensorrt":
+                    warnings.warn(
+                        "`paddle_tensorrt` is deprecated. Please use `paddle_infer` instead.",
+                        FutureWarning,
+                    )
+                    new_data[device_type] = "paddle_infer"
+        return new_data
+
+    @field_validator("backend_configs", mode="before")
+    @classmethod
+    def _hack_backend_configs(cls, data: Any) -> Any:
+        if isinstance(data, Mapping):
+            new_data = dict(data)
+            if new_data and "paddle_tensorrt" in new_data:
+                warnings.warn(
+                    "`paddle_tensorrt` is deprecated. Please use `paddle_infer` instead.",
+                    FutureWarning,
+                )
+                if "paddle_infer" not in new_data:
+                    new_data["paddle_infer"] = {}
+                pptrt_cfg = new_data["paddle_tensorrt"]
+                logging.warning("`paddle_infer.enable_trt` will be set to `True`.")
+                new_data["paddle_infer"]["enable_trt"] = True
+                new_data["paddle_infer"]["trt_dynamic_shapes"] = pptrt_cfg[
+                    "dynamic_shapes"
+                ]
+                if "dynamic_shape_input_data" in pptrt_cfg:
+                    new_data["paddle_infer"]["trt_dynamic_shape_input_data"] = (
+                        pptrt_cfg["dynamic_shape_input_data"]
+                    )
+                logging.warning("`paddle_tensorrt.enable_log_info` will be ignored.")
+        return new_data

libs/paddlex-hpi/src/paddlex_hpi/_config.py

@@ -0,0 +1,59 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import platform
+from functools import lru_cache
+from typing import Optional
+
+from importlib_resources import files
+from paddlex.utils import logging
+
+from paddlex_hpi._utils.typing import DeviceType
+
+_DB_PATH: str = "model_info_collection.json"
+
+
+@lru_cache(1)
+def _get_model_info_collection() -> dict:
+    with files("paddlex_hpi").joinpath(_DB_PATH).open("r", encoding="utf-8") as f:
+        _model_info_collection = json.load(f)
+    return _model_info_collection
+
+
+def get_model_info(model_name: str, device_type: DeviceType) -> Optional[dict]:
+    # TODO: Typed model info and nearest referents
+    model_info_collection = _get_model_info_collection()
+    uname = platform.uname()
+    arch = uname.machine.lower()
+    if arch not in model_info_collection:
+        return None
+    logging.debug("Getting model information for arch: %s", arch)
+    model_info_collection = model_info_collection[arch]
+    os = uname.system.lower()
+    if os not in model_info_collection:
+        return None
+    logging.debug("Getting model information for OS: %s", os)
+    model_info_collection = model_info_collection[os]
+    if device_type == "cpu":
+        device = "cpu"
+    elif device_type == "gpu":
+        device = "gpu_cuda118_cudnn86"
+    else:
+        return None
+    logging.debug("Getting model information for device: %s", device)
+    model_info_collection = model_info_collection[device]
+    if model_name not in model_info_collection:
+        return None
+    return model_info_collection[model_name]