From 9224356d7f29edb087516e7a02dd326954c3e9a9 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Mon, 30 Jun 2025 13:54:27 +0800 Subject: [PATCH 1/2] support dynamic shape when use CINN --- .pre-commit-config.yaml | 52 +++++++++--------- deepmd/pd/train/training.py | 76 +++++++++++++------------ deepmd/pd/utils/env.py | 8 +++ doc/html/_sources/env.md | 107 ++++++++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 60 deletions(-) create mode 100644 doc/html/_sources/env.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba799dc4ac..5385f80097 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,13 +65,13 @@ repos: - id: clang-format exclude: ^(source/3rdparty|source/lib/src/gpu/cudart/.+\.inc|.+\.ipynb$|.+\.json$) # markdown, yaml, CSS, javascript - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 - hooks: - - id: prettier - types_or: [markdown, yaml, css] - # workflow files cannot be modified by pre-commit.ci - exclude: ^(source/3rdparty|\.github/workflows|\.clang-format) + # - repo: https://github.com/pre-commit/mirrors-prettier + # rev: v4.0.0-alpha.8 + # hooks: + # - id: prettier + # types_or: [markdown, yaml, css] + # # workflow files cannot be modified by pre-commit.ci + # exclude: ^(source/3rdparty|\.github/workflows|\.clang-format) # Shell - repo: https://github.com/scop/pre-commit-shfmt rev: v3.11.0-1 @@ -83,25 +83,25 @@ repos: hooks: - id: cmake-format #- id: cmake-lint - - repo: https://github.com/njzjz/mirrors-bibtex-tidy - rev: v1.13.0 - hooks: - - id: bibtex-tidy - args: - - --curly - - --numeric - - --align=13 - - --blank-lines - # disable sort: the order of keys and fields has explict meanings - #- --sort=key - - --duplicates=key,doi,citation,abstract - - --merge=combine - #- --sort-fields - #- --strip-comments - - --trailing-commas - - --encode-urls - - --remove-empty-fields - - --wrap=80 + # - repo: https://github.com/njzjz/mirrors-bibtex-tidy + # rev: v1.13.0 + # hooks: + # - id: bibtex-tidy + # args: + # - --curly + # - --numeric + # - --align=13 + # - --blank-lines + # # disable sort: the order of keys and fields has explict meanings + # #- --sort=key + # - --duplicates=key,doi,citation,abstract + # - --merge=combine + # #- --sort-fields + # #- --strip-comments + # - --trailing-commas + # - --encode-urls + # - --remove-empty-fields + # - --wrap=80 # license header - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 diff --git a/deepmd/pd/train/training.py b/deepmd/pd/train/training.py index ba3531bc8a..13446db89d 100644 --- a/deepmd/pd/train/training.py +++ b/deepmd/pd/train/training.py @@ -54,6 +54,7 @@ ) from deepmd.pd.utils.env import ( CINN, + CINN_ALLOW_DYNAMIC_SHAPE, DEFAULT_PRECISION, DEVICE, JIT, @@ -612,41 +613,48 @@ def warm_up_linear(step, warmup_steps): # NOTE: This is a trick to decide the right input_spec for wrapper.forward _, label_dict, _ = self.get_data(is_train=True) - # Define specification templates - spec_templates = { - "find_box": np.float32(1.0), - "find_coord": np.float32(1.0), - "find_numb_copy": np.float32(0.0), - "numb_copy": static.InputSpec([1, 1], "int64", name="numb_copy"), - "find_energy": np.float32(1.0), - "energy": static.InputSpec([1, 1], "float64", name="energy"), - "find_force": np.float32(1.0), - "force": static.InputSpec([1, -1, 3], "float64", name="force"), - "find_virial": np.float32(0.0), - "virial": static.InputSpec([1, 9], "float64", name="virial"), - "natoms": static.InputSpec([1, -1], "int32", name="natoms"), - } # Build spec only for keys present in sample data - label_dict_spec = { - k: spec_templates[k] for k in label_dict.keys() if k in spec_templates - } - self.wrapper.forward = jit.to_static( - backend=backend, - input_spec=[ - static.InputSpec([1, -1, 3], "float64", name="coord"), # coord - static.InputSpec([1, -1], "int32", name="atype"), # atype - None, # spin - static.InputSpec([1, 9], "float64", name="box"), # box - static.InputSpec([], "float64", name="cur_lr"), # cur_lr - label_dict_spec, # label, - # None, # task_key - # False, # inference_only - # False, # do_atomic_virial - # None, # fparam - # None, # aparam - ], - full_graph=True, - )(self.wrapper.forward) + if CINN_ALLOW_DYNAMIC_SHAPE: + # Define specification templates + spec_templates = { + "find_box": np.float32(1.0), + "find_coord": np.float32(1.0), + "find_numb_copy": np.float32(0.0), + "numb_copy": static.InputSpec([1, 1], "int64", name="numb_copy"), + "find_energy": np.float32(1.0), + "energy": static.InputSpec([1, 1], "float64", name="energy"), + "find_force": np.float32(1.0), + "force": static.InputSpec([1, -1, 3], "float64", name="force"), + "find_virial": np.float32(0.0), + "virial": static.InputSpec([1, 9], "float64", name="virial"), + "natoms": static.InputSpec([1, -1], "int32", name="natoms"), + } + label_dict_spec = { + k: spec_templates[k] + for k in label_dict.keys() + if k in spec_templates + } + self.wrapper.forward = jit.to_static( + backend=None, + input_spec=[ + static.InputSpec([1, -1, 3], "float64", name="coord"), # coord + static.InputSpec([1, -1], "int32", name="atype"), # atype + None, # spin + static.InputSpec([1, 9], "float64", name="box"), # box + static.InputSpec([], "float64", name="cur_lr"), # cur_lr + label_dict_spec, # label, + # None, # task_key + # False, # inference_only + # False, # do_atomic_virial + # None, # fparam + # None, # aparam + ], + full_graph=True, + )(self.wrapper.forward) + else: + self.wrapper.forward = jit.to_static(full_graph=True, backend=backend)( + self.wrapper.forward + ) log.info( "Enable CINN during training, there may be some additional " diff --git a/deepmd/pd/utils/env.py b/deepmd/pd/utils/env.py index c6b9b4eab5..149f56c587 100644 --- a/deepmd/pd/utils/env.py +++ b/deepmd/pd/utils/env.py @@ -69,6 +69,14 @@ def to_bool(flag: int | bool | str) -> bool: "installation or recompiling with CINN enabled." ) +# NOTE: Allow the CINN compiler to optimize inputs with dynamic shapes, +# may lead to a slight performance decrease compared to static shapes. + +# If you can confirm that the shape of the input tensors will not change, +# you can set it to False to further enhance performance. +# Otherwise, please use the default value(True) to improve runtime compatibility. +CINN_ALLOW_DYNAMIC_SHAPE = to_bool(os.environ.get("CINN_ALLOW_DYNAMIC_SHAPE", True)) + CACHE_PER_SYS = 5 # keep at most so many sets per sys in memory ENERGY_BIAS_TRAINABLE = True CUSTOM_OP_USE_JIT = to_bool(os.environ.get("CUSTOM_OP_USE_JIT", False)) diff --git a/doc/html/_sources/env.md b/doc/html/_sources/env.md new file mode 100644 index 0000000000..29a68fe8e8 --- /dev/null +++ b/doc/html/_sources/env.md @@ -0,0 +1,107 @@ +# Runtime environment variables + +:::{note} +For build-time environment variables, see [Install from source code](./install/install-from-source.md). +::: + +## All interfaces + +:::{envvar} DP_INTER_OP_PARALLELISM_THREADS + +**Alias**: `TF_INTER_OP_PARALLELISM_THREADS` +**Default**: `0` + +Control parallelism within TensorFlow (when TensorFlow is built against Eigen) and PyTorch native OPs for CPU devices. +See [How to control the parallelism of a job](./troubleshooting/howtoset_num_nodes.md) for details. +::: + +:::{envvar} DP_INTRA_OP_PARALLELISM_THREADS + +**Alias**: `TF_INTRA_OP_PARALLELISM_THREADS`\*\* +**Default**: `0` + +Control parallelism within TensorFlow (when TensorFlow is built against Eigen) and PyTorch native OPs. +See [How to control the parallelism of a job](./troubleshooting/howtoset_num_nodes.md) for details. +::: + +## Environment variables of dependencies + +- If OpenMP is used, [OpenMP environment variables](https://www.openmp.org/spec-html/5.0/openmpch6.html) can be used to control OpenMP threads, such as [`OMP_NUM_THREADS`](https://www.openmp.org/spec-html/5.0/openmpse50.html#x289-20540006.2). +- If CUDA is used, [CUDA environment variables](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#cuda-environment-variables) can be used to control CUDA devices, such as `CUDA_VISIBLE_DEVICES`. +- If ROCm is used, [ROCm environment variables](https://rocm.docs.amd.com/en/latest/conceptual/gpu-isolation.html#environment-variables) can be used to control ROCm devices. +- {{ tensorflow_icon }} If TensorFlow is used, TensorFlow environment variables can be used. +- {{ pytorch_icon }} If PyTorch is used, [PyTorch environment variables](https://pytorch.org/docs/stable/torch_environment_variables.html) can be used. +- {{ jax_icon }} [`JAX_PLATFORMS`](https://jax.readthedocs.io/en/latest/faq.html#controlling-data-and-computation-placement-on-devices) and [`XLA_FLAGS`](https://jax.readthedocs.io/en/latest/gpu_performance_tips.html#xla-performance-flags) are commonly used. +- {{ paddle_icon }} If Paddle is used, [Padddle environment variables](https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/guides/flags/flags_cn.html) can be used. + +## Python interface only + +:::{envvar} DP_INTERFACE_PREC + +**Choices**: `high`, `low`; **Default**: `high` + +Control high (double) or low (float) precision of training. +::: + +:::{envvar} DP_AUTO_PARALLELIZATION + +**Choices**: `0`, `1`; **Default**: `0` + +{{ tensorflow_icon }} Enable auto parallelization for CPU operators. +::: + +:::{envvar} DP_JIT + +**Choices**: `0`, `1`; **Default**: `0` + +{{ tensorflow_icon }} Enable JIT. Note that this option may either improve or decrease the performance. Requires TensorFlow to support JIT. +::: + +:::{envvar} CINN + +**Choices**: `0`, `1`; **Default**: `0` + +{{ tensorflow_icon }} Enable Paddle CINN Compiler when `PD_JIT` is enabled. Note that this option may either improve or decrease the performance. Requires Paddle to support CINN()(`paddle.device.is_compiled_with_cinn()` is `True`). +::: + +:::{envvar} CINN_ALLOW_DYNAMIC_SHAPE + +**Choices**: `0`, `1`; **Default**: `0` + +{{ tensorflow_icon }} Allow the CINN compiler to optimize inputs with dynamic shapes, may lead to a slight performance decrease compared to static shapes. +::: + +:::{envvar} DP_INFER_BATCH_SIZE + +**Default**: `1024` on CPUs and as maximum as possible until out-of-memory on GPUs + +Inference batch size, calculated by multiplying the number of frames with the number of atoms. +::: + +:::{envvar} DP_BACKEND + +**Default**: `tensorflow` + +Default backend. +::: + +:::{envvar} NUM_WORKERS + +**Default**: 8 or the number of cores (whichever is smaller) + +{{ pytorch_icon }} Number of subprocesses to use for data loading in the PyTorch backend. +See [PyTorch documentation](https://pytorch.org/docs/stable/data.html) for details. + +::: + +## C++ interface only + +These environment variables also apply to third-party programs using the C++ interface, such as [LAMMPS](./third-party/lammps-command.md). + +:::{envvar} DP_PLUGIN_PATH + +**Type**: List of paths, split by `:` on Unix and `;` on Windows + +List of customized OP plugin libraries to load, such as `/path/to/plugin1.so:/path/to/plugin2.so` on Linux and `/path/to/plugin1.dll;/path/to/plugin2.dll` on Windows. + +::: From 0524f5cb91ea151343e54111e2c70229650746ec Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Mon, 30 Jun 2025 13:55:23 +0800 Subject: [PATCH 2/2] support dynamic shape when use CINN --- .pre-commit-config.yaml | 52 +++++++++--------- deepmd/main.py | 4 +- deepmd/pd/train/training.py | 23 +++++--- deepmd/pd/utils/env.py | 1 + doc/html/_sources/env.md | 107 ------------------------------------ doc/train/training.md | 3 + 6 files changed, 48 insertions(+), 142 deletions(-) delete mode 100644 doc/html/_sources/env.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5385f80097..ba799dc4ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,13 +65,13 @@ repos: - id: clang-format exclude: ^(source/3rdparty|source/lib/src/gpu/cudart/.+\.inc|.+\.ipynb$|.+\.json$) # markdown, yaml, CSS, javascript - # - repo: https://github.com/pre-commit/mirrors-prettier - # rev: v4.0.0-alpha.8 - # hooks: - # - id: prettier - # types_or: [markdown, yaml, css] - # # workflow files cannot be modified by pre-commit.ci - # exclude: ^(source/3rdparty|\.github/workflows|\.clang-format) + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + types_or: [markdown, yaml, css] + # workflow files cannot be modified by pre-commit.ci + exclude: ^(source/3rdparty|\.github/workflows|\.clang-format) # Shell - repo: https://github.com/scop/pre-commit-shfmt rev: v3.11.0-1 @@ -83,25 +83,25 @@ repos: hooks: - id: cmake-format #- id: cmake-lint - # - repo: https://github.com/njzjz/mirrors-bibtex-tidy - # rev: v1.13.0 - # hooks: - # - id: bibtex-tidy - # args: - # - --curly - # - --numeric - # - --align=13 - # - --blank-lines - # # disable sort: the order of keys and fields has explict meanings - # #- --sort=key - # - --duplicates=key,doi,citation,abstract - # - --merge=combine - # #- --sort-fields - # #- --strip-comments - # - --trailing-commas - # - --encode-urls - # - --remove-empty-fields - # - --wrap=80 + - repo: https://github.com/njzjz/mirrors-bibtex-tidy + rev: v1.13.0 + hooks: + - id: bibtex-tidy + args: + - --curly + - --numeric + - --align=13 + - --blank-lines + # disable sort: the order of keys and fields has explict meanings + #- --sort=key + - --duplicates=key,doi,citation,abstract + - --merge=combine + #- --sort-fields + #- --strip-comments + - --trailing-commas + - --encode-urls + - --remove-empty-fields + - --wrap=80 # license header - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 diff --git a/deepmd/main.py b/deepmd/main.py index 14c0390bdc..492f3b085e 100644 --- a/deepmd/main.py +++ b/deepmd/main.py @@ -112,7 +112,7 @@ def main_parser() -> argparse.ArgumentParser: if default_backend not in BACKEND_TABLE.keys(): raise ValueError( f"Unknown backend {default_backend}. " - "Please set DP_BACKEND to either tensorflow or pytorch." + "Please set DP_BACKEND to either tensorflow, pytorch, or paddle." ) parser_backend = parser.add_mutually_exclusive_group() @@ -312,7 +312,7 @@ def main_parser() -> argparse.ArgumentParser: "--output", type=str, default="frozen_model", - help="Filename (prefix) of the output model file. TensorFlow backend: suffix is .pb; PyTorch backend: suffix is .pth", + help="Filename (prefix) of the output model file. TensorFlow backend: suffix is .pb; PyTorch backend: suffix is .pth; Paddle backend: suffix is .json and .pdiparams", ) parser_frz.add_argument( "-n", diff --git a/deepmd/pd/train/training.py b/deepmd/pd/train/training.py index 13446db89d..44d0d16439 100644 --- a/deepmd/pd/train/training.py +++ b/deepmd/pd/train/training.py @@ -610,11 +610,10 @@ def warm_up_linear(step, warmup_steps): ) backend = "CINN" if CINN else None - # NOTE: This is a trick to decide the right input_spec for wrapper.forward - _, label_dict, _ = self.get_data(is_train=True) - - # Build spec only for keys present in sample data if CINN_ALLOW_DYNAMIC_SHAPE: + # Build spec only for keys present in sample data + # NOTE: This is a trick to decide the right input_spec for wrapper.forward + _, label_dict, _ = self.get_data(is_train=True) # Define specification templates spec_templates = { "find_box": np.float32(1.0), @@ -635,7 +634,7 @@ def warm_up_linear(step, warmup_steps): if k in spec_templates } self.wrapper.forward = jit.to_static( - backend=None, + backend=backend, input_spec=[ static.InputSpec([1, -1, 3], "float64", name="coord"), # coord static.InputSpec([1, -1], "int32", name="atype"), # atype @@ -657,9 +656,19 @@ def warm_up_linear(step, warmup_steps): ) log.info( - "Enable CINN during training, there may be some additional " - "compilation time in the first traning step." + "[CINN] Enable CINN during training, there may be some additional " + "compilation time in the first training step." ) + if not CINN_ALLOW_DYNAMIC_SHAPE: + log.info( + "[CINN] Dynamic shape is disabled (CINN_ALLOW_DYNAMIC_SHAPE=0). " + "Make sure the input batch shapes are fixed during training. " + "This is recommended for optimal performance, e.g., as in examples/water." + ) + log.info( + "[CINN] If batch data from your dataset(s) has varying input shapes, consider setting " + "CINN_ALLOW_DYNAMIC_SHAPE=1 to enable dynamic shape support." + ) if dist.is_available() and dist.is_initialized(): # DDP will guarantee the model parameters are identical across all processes diff --git a/deepmd/pd/utils/env.py b/deepmd/pd/utils/env.py index 149f56c587..28606d0945 100644 --- a/deepmd/pd/utils/env.py +++ b/deepmd/pd/utils/env.py @@ -207,6 +207,7 @@ def enable_prim(enable: bool = True): __all__ = [ "CACHE_PER_SYS", "CINN", + "CINN_ALLOW_DYNAMIC_SHAPE", "CUSTOM_OP_USE_JIT", "DEFAULT_PRECISION", "DEVICE", diff --git a/doc/html/_sources/env.md b/doc/html/_sources/env.md deleted file mode 100644 index 29a68fe8e8..0000000000 --- a/doc/html/_sources/env.md +++ /dev/null @@ -1,107 +0,0 @@ -# Runtime environment variables - -:::{note} -For build-time environment variables, see [Install from source code](./install/install-from-source.md). -::: - -## All interfaces - -:::{envvar} DP_INTER_OP_PARALLELISM_THREADS - -**Alias**: `TF_INTER_OP_PARALLELISM_THREADS` -**Default**: `0` - -Control parallelism within TensorFlow (when TensorFlow is built against Eigen) and PyTorch native OPs for CPU devices. -See [How to control the parallelism of a job](./troubleshooting/howtoset_num_nodes.md) for details. -::: - -:::{envvar} DP_INTRA_OP_PARALLELISM_THREADS - -**Alias**: `TF_INTRA_OP_PARALLELISM_THREADS`\*\* -**Default**: `0` - -Control parallelism within TensorFlow (when TensorFlow is built against Eigen) and PyTorch native OPs. -See [How to control the parallelism of a job](./troubleshooting/howtoset_num_nodes.md) for details. -::: - -## Environment variables of dependencies - -- If OpenMP is used, [OpenMP environment variables](https://www.openmp.org/spec-html/5.0/openmpch6.html) can be used to control OpenMP threads, such as [`OMP_NUM_THREADS`](https://www.openmp.org/spec-html/5.0/openmpse50.html#x289-20540006.2). -- If CUDA is used, [CUDA environment variables](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#cuda-environment-variables) can be used to control CUDA devices, such as `CUDA_VISIBLE_DEVICES`. -- If ROCm is used, [ROCm environment variables](https://rocm.docs.amd.com/en/latest/conceptual/gpu-isolation.html#environment-variables) can be used to control ROCm devices. -- {{ tensorflow_icon }} If TensorFlow is used, TensorFlow environment variables can be used. -- {{ pytorch_icon }} If PyTorch is used, [PyTorch environment variables](https://pytorch.org/docs/stable/torch_environment_variables.html) can be used. -- {{ jax_icon }} [`JAX_PLATFORMS`](https://jax.readthedocs.io/en/latest/faq.html#controlling-data-and-computation-placement-on-devices) and [`XLA_FLAGS`](https://jax.readthedocs.io/en/latest/gpu_performance_tips.html#xla-performance-flags) are commonly used. -- {{ paddle_icon }} If Paddle is used, [Padddle environment variables](https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/guides/flags/flags_cn.html) can be used. - -## Python interface only - -:::{envvar} DP_INTERFACE_PREC - -**Choices**: `high`, `low`; **Default**: `high` - -Control high (double) or low (float) precision of training. -::: - -:::{envvar} DP_AUTO_PARALLELIZATION - -**Choices**: `0`, `1`; **Default**: `0` - -{{ tensorflow_icon }} Enable auto parallelization for CPU operators. -::: - -:::{envvar} DP_JIT - -**Choices**: `0`, `1`; **Default**: `0` - -{{ tensorflow_icon }} Enable JIT. Note that this option may either improve or decrease the performance. Requires TensorFlow to support JIT. -::: - -:::{envvar} CINN - -**Choices**: `0`, `1`; **Default**: `0` - -{{ tensorflow_icon }} Enable Paddle CINN Compiler when `PD_JIT` is enabled. Note that this option may either improve or decrease the performance. Requires Paddle to support CINN()(`paddle.device.is_compiled_with_cinn()` is `True`). -::: - -:::{envvar} CINN_ALLOW_DYNAMIC_SHAPE - -**Choices**: `0`, `1`; **Default**: `0` - -{{ tensorflow_icon }} Allow the CINN compiler to optimize inputs with dynamic shapes, may lead to a slight performance decrease compared to static shapes. -::: - -:::{envvar} DP_INFER_BATCH_SIZE - -**Default**: `1024` on CPUs and as maximum as possible until out-of-memory on GPUs - -Inference batch size, calculated by multiplying the number of frames with the number of atoms. -::: - -:::{envvar} DP_BACKEND - -**Default**: `tensorflow` - -Default backend. -::: - -:::{envvar} NUM_WORKERS - -**Default**: 8 or the number of cores (whichever is smaller) - -{{ pytorch_icon }} Number of subprocesses to use for data loading in the PyTorch backend. -See [PyTorch documentation](https://pytorch.org/docs/stable/data.html) for details. - -::: - -## C++ interface only - -These environment variables also apply to third-party programs using the C++ interface, such as [LAMMPS](./third-party/lammps-command.md). - -:::{envvar} DP_PLUGIN_PATH - -**Type**: List of paths, split by `:` on Unix and `;` on Windows - -List of customized OP plugin libraries to load, such as `/path/to/plugin1.so:/path/to/plugin2.so` on Linux and `/path/to/plugin1.dll;/path/to/plugin2.dll` on Windows. - -::: diff --git a/doc/train/training.md b/doc/train/training.md index 6c8b7a5549..ca0b46c0ef 100644 --- a/doc/train/training.md +++ b/doc/train/training.md @@ -34,7 +34,10 @@ $ dp --pd train input.json # [experimental] training model with CINN compiler for better performance, # see: https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/guides/paddle_v3_features/cinn_cn.html +## If the shape(s) of batch input data are dynamic during training(default). $ CINN=1 dp --pd train input.json +## If the shape(s) of batch input data are fixed during training, e.g., examples/water. +$ CINN=1 CINN_ALLOW_DYNAMIC_SHAPE=0 dp --pd train input.json ``` :::