lazy import paddle to avoid conflicts with HPI

TingquanGao · TingquanGao · commit 120ade1531d4 · 2024-09-23T02:30:43.000+08:00
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -22,7 +22,7 @@ repos:
     -   id: clang-format
         name: clang-format
         description: Format files with ClangFormat
-        entry: bash .clang_format.hook -i
+        entry: bash .precommit/clang_format.hook -i
         language: system
         files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$
 # For Python files
@@ -46,8 +46,8 @@ repos:
 # check license
 -   repo: local
     hooks:
-    -   id: check-license
-        name: Check License
-        entry: python .check_license.py
+    -   id: check-custom
+        name: Check Custom
+        entry: python .precommit/check_custom.py
         language: python
         files: \.py$
diff --git a/.precommit/check_custom.py b/.precommit/check_custom.py
@@ -31,20 +31,23 @@
 """
 
 
-def check_license(file_path):
+def check(file_path):
     with open(file_path, "r") as f:
         content = f.read()
-        if not content.startswith(LICENSE_TEXT):
-            print(f"License header missing in {file_path}")
-            return False
+    if not content.startswith(LICENSE_TEXT):
+        print(f"License header missing in {file_path}")
+        return False
+    if "import paddle" in content or "from paddle import " in content:
+        print(f"Please using `lazy_paddle` instead `paddle` when import in {file_path}")
+        return False
     return True
 
 
 def main():
     files = sys.argv[1:]
     all_files_valid = True
     for file in files:
-        if not check_license(file):
+        if not check(file):
             all_files_valid = False
     if not all_files_valid:
         sys.exit(1)
diff --git a/.precommit/clang_format.hook b/.precommit/clang_format.hook
diff --git a/paddlex/__init__.py b/paddlex/__init__.py
@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .utils.lazy_loader import LazyLoader
+import sys
+
+sys.modules["lazy_paddle"] = LazyLoader("lazy_paddle", globals(), "paddle")
+
 import os
 
 from . import version
diff --git a/paddlex/inference/components/paddle_predictor/predictor.py b/paddlex/inference/components/paddle_predictor/predictor.py
@@ -14,9 +14,7 @@
 
 import os
 from abc import abstractmethod
-import numpy as np
-import paddle
-from paddle.inference import Config, create_predictor
+import lazy_paddle as paddle
 import numpy as np
 
 from ..base import BaseComponent
@@ -44,6 +42,8 @@ def __init__(self, model_dir, model_prefix, option):
 
     def _create(self, model_dir, model_prefix, option):
         """_create"""
+        from lazy_paddle.inference import Config, create_predictor
+
         use_pir = (
             hasattr(paddle.framework, "use_pir_api") and paddle.framework.use_pir_api()
         )
diff --git a/paddlex/inference/components/task_related/text_rec.py b/paddlex/inference/components/task_related/text_rec.py
@@ -21,7 +21,7 @@
 from PIL import Image
 import cv2
 import math
-import paddle
+import lazy_paddle as paddle
 import json
 import tempfile
 from tokenizers import Tokenizer as TokenizerFast
diff --git a/paddlex/modules/base/predictor/utils/paddle_inference_predictor.py b/paddlex/modules/base/predictor/utils/paddle_inference_predictor.py
@@ -13,8 +13,7 @@
 # limitations under the License.
 
 import os
-import paddle
-from paddle.inference import Config, create_predictor
+import lazy_paddle as paddle
 
 from .....utils import logging
 
@@ -34,6 +33,8 @@ def __init__(self, model_dir, model_prefix, option, delete_pass=[]):
 
     def _create(self, model_dir, model_prefix, option, delete_pass):
         """_create"""
+        from lazy_paddle.inference import Config, create_predictor
+
         use_pir = (
             hasattr(paddle.framework, "use_pir_api") and paddle.framework.use_pir_api()
         )
diff --git a/paddlex/modules/base/trainer/train_deamon.py b/paddlex/modules/base/trainer/train_deamon.py
@@ -20,7 +20,7 @@
 import threading
 from abc import ABC, abstractmethod
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..build_model import build_model
 from ....utils.file_interface import write_json_file
diff --git a/paddlex/modules/base/utils/topk_eval.py b/paddlex/modules/base/utils/topk_eval.py
@@ -16,8 +16,7 @@
 import os
 import json
 import argparse
-from paddle import nn
-import paddle
+import lazy_paddle as paddle
 
 from ....utils import logging
 
@@ -34,7 +33,7 @@ def parse_args():
     return args
 
 
-class AvgMetrics(nn.Layer):
+class AvgMetrics(paddle.nn.Layer):
     """Average metrics"""
 
     def __init__(self):
diff --git a/paddlex/modules/image_classification/trainer.py b/paddlex/modules/image_classification/trainer.py
@@ -14,7 +14,7 @@
 
 import json
 import shutil
-import paddle
+import lazy_paddle as paddle
 from pathlib import Path
 
 from ..base import BaseTrainer, BaseTrainDeamon
diff --git a/paddlex/modules/multilabel_classification/trainer.py b/paddlex/modules/multilabel_classification/trainer.py
@@ -14,7 +14,7 @@
 
 import json
 import shutil
-import paddle
+import lazy_paddle as paddle
 from pathlib import Path
 
 from ..base import BaseTrainer, BaseTrainDeamon
@@ -48,34 +48,32 @@ def build_deamon(self, config: AttrDict) -> "ClsTrainDeamon":
         return ClsTrainDeamon(config)
 
     def update_config(self):
-        """update training config
-        """
+        """update training config"""
         if self.train_config.log_interval:
             self.pdx_config.update_log_interval(self.train_config.log_interval)
         if self.train_config.eval_interval:
-            self.pdx_config.update_eval_interval(
-                self.train_config.eval_interval)
+            self.pdx_config.update_eval_interval(self.train_config.eval_interval)
         if self.train_config.save_interval:
-            self.pdx_config.update_save_interval(
-                self.train_config.save_interval)
+            self.pdx_config.update_save_interval(self.train_config.save_interval)
 
-        self.pdx_config.update_dataset(self.global_config.dataset_dir,
-                                       "MLClsDataset")
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "MLClsDataset")
         if self.train_config.num_classes is not None:
             self.pdx_config.update_num_classes(self.train_config.num_classes)
-        if self.train_config.pretrain_weight_path and self.train_config.pretrain_weight_path != "":
+        if (
+            self.train_config.pretrain_weight_path
+            and self.train_config.pretrain_weight_path != ""
+        ):
             self.pdx_config.update_pretrained_weights(
-                self.train_config.pretrain_weight_path)
+                self.train_config.pretrain_weight_path
+            )
 
-        label_dict_path = Path(self.global_config.dataset_dir).joinpath(
-            "label.txt")
+        label_dict_path = Path(self.global_config.dataset_dir).joinpath("label.txt")
         if label_dict_path.exists():
             self.dump_label_dict(label_dict_path)
         if self.train_config.batch_size is not None:
             self.pdx_config.update_batch_size(self.train_config.batch_size)
         if self.train_config.learning_rate is not None:
-            self.pdx_config.update_learning_rate(
-                self.train_config.learning_rate)
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
         if self.train_config.epochs_iters is not None:
             self.pdx_config._update_epochs(self.train_config.epochs_iters)
         if self.train_config.warmup_steps is not None:
diff --git a/paddlex/modules/object_detection/trainer.py b/paddlex/modules/object_detection/trainer.py
@@ -14,7 +14,7 @@
 
 
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
diff --git a/paddlex/modules/semantic_segmentation/trainer.py b/paddlex/modules/semantic_segmentation/trainer.py
@@ -16,7 +16,7 @@
 import os
 import glob
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
diff --git a/paddlex/modules/table_recognition/predictor/transforms.py b/paddlex/modules/table_recognition/predictor/transforms.py
@@ -17,7 +17,7 @@
 import os.path as osp
 import numpy as np
 import cv2
-import paddle
+import lazy_paddle as paddle
 
 from .keys import TableRecKeys as K
 from ...base import BaseTransform
diff --git a/paddlex/modules/table_recognition/trainer.py b/paddlex/modules/table_recognition/trainer.py
@@ -15,7 +15,7 @@
 
 import os
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
@@ -76,7 +76,10 @@ def get_train_kwargs(self) -> dict:
         Returns:
             dict: the arguments of training function.
         """
-        return {"device": self.get_device(), "dy2st": self.train_config.get("dy2st", False)}
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+        }
 
 
 class TableRecTrainDeamon(BaseTrainDeamon):
diff --git a/paddlex/modules/text_detection/trainer.py b/paddlex/modules/text_detection/trainer.py
@@ -15,7 +15,7 @@
 
 import os
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
@@ -74,7 +74,10 @@ def get_train_kwargs(self) -> dict:
         Returns:
             dict: the arguments of training function.
         """
-        return {"device": self.get_device(), "dy2st": self.train_config.get("dy2st", False)}
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+        }
 
 
 class TextDetTrainDeamon(BaseTrainDeamon):
diff --git a/paddlex/modules/text_recognition/dataset_checker/dataset_src/convert_dataset.py b/paddlex/modules/text_recognition/dataset_checker/dataset_src/convert_dataset.py
@@ -21,7 +21,7 @@
 import pickle
 from tqdm import tqdm
 from collections import defaultdict
-from paddle.utils import try_import
+import imagesize
 from .....utils.errors import ConvertFailedError
 from .....utils.logging import info, warning
 
@@ -66,7 +66,6 @@ def convert_pkl_dataset(root_dir):
 
 
 def txt2pickle(images, equations, save_dir):
-    imagesize = try_import("imagesize")
     phase = os.path.basename(equations).replace(".txt", "")
     save_p = os.path.join(save_dir, "latexocr_{}.pkl".format(phase))
     min_dimensions = (32, 32)
diff --git a/paddlex/modules/text_recognition/predictor/transforms.py b/paddlex/modules/text_recognition/predictor/transforms.py
@@ -21,7 +21,7 @@
 from PIL import Image
 import cv2
 import math
-import paddle
+import lazy_paddle as paddle
 import json
 import tempfile
 from tokenizers import Tokenizer as TokenizerFast
diff --git a/paddlex/modules/text_recognition/trainer.py b/paddlex/modules/text_recognition/trainer.py
@@ -16,7 +16,7 @@
 import os
 import shutil
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
@@ -108,7 +108,10 @@ def get_train_kwargs(self) -> dict:
         Returns:
             dict: the arguments of training function.
         """
-        return {"device": self.get_device(), "dy2st": self.train_config.get("dy2st", False)}
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+        }
 
 
 class TextRecTrainDeamon(BaseTrainDeamon):
diff --git a/paddlex/modules/ts_anomaly_detection/trainer.py b/paddlex/modules/ts_anomaly_detection/trainer.py
@@ -17,7 +17,7 @@
 import time
 from pathlib import Path
 import tarfile
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
diff --git a/paddlex/modules/ts_classification/trainer.py b/paddlex/modules/ts_classification/trainer.py
@@ -17,7 +17,7 @@
 import time
 import tarfile
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
diff --git a/paddlex/modules/ts_forecast/trainer.py b/paddlex/modules/ts_forecast/trainer.py
@@ -17,7 +17,7 @@
 import time
 import tarfile
 from pathlib import Path
-import paddle
+import lazy_paddle as paddle
 
 from ..base import BaseTrainer, BaseTrainDeamon
 from ...utils.config import AttrDict
diff --git a/paddlex/pipelines/OCR/utils.py b/paddlex/pipelines/OCR/utils.py
@@ -21,7 +21,7 @@
 import math
 import copy
 
-from paddlex.utils.fonts import PINGFANG_FONT_FILE_PATH
+from ...utils.fonts import PINGFANG_FONT_FILE_PATH
 
 
 def draw_ocr_box_txt(
diff --git a/paddlex/repo_apis/PaddleClas_api/cls/config.py b/paddlex/repo_apis/PaddleClas_api/cls/config.py
@@ -14,7 +14,6 @@
 
 import yaml
 from typing import Union
-from paddleclas.ppcls.utils.config import get_config, override_config
 
 from ...base import BaseConfig
 from ....utils.misc import abspath
@@ -33,6 +32,8 @@ def update(self, list_like_obj: list):
                     'VALID.transforms.1.ResizeImage.resize_short=300'
                 ]
         """
+        from paddleclas.ppcls.utils.config import override_config
+
         dict_ = override_config(self.dict, list_like_obj)
         self.reset_from_dict(dict_)
 
diff --git a/paddlex/repo_apis/PaddleSeg_api/base_seg_config.py b/paddlex/repo_apis/PaddleSeg_api/base_seg_config.py
diff --git a/paddlex/repo_apis/PaddleSeg_api/seg/config.py b/paddlex/repo_apis/PaddleSeg_api/seg/config.py
diff --git a/paddlex/repo_apis/PaddleTS_api/ts_base/config.py b/paddlex/repo_apis/PaddleTS_api/ts_base/config.py
diff --git a/paddlex/utils/device.py b/paddlex/utils/device.py
diff --git a/paddlex/utils/lazy_loader.py b/paddlex/utils/lazy_loader.py
diff --git a/requirements.txt b/requirements.txt
