From 61ffd154c7381e7a7fa897c56fcf8a3d0ef2e3c4 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 21 Apr 2024 00:53:44 -0400 Subject: [PATCH 01/17] ci(build): use uv for cibuildwheel Signed-off-by: Jinzhe Zeng --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index edbc5e0b19..62f4679986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,7 @@ test-command = [ "pytest {project}/source/tests/tf/test_lammps.py" ] test-extras = ["cpu", "test", "lmp", "ipi"] +before-test = ["pip install replace-pip-with-uv-pip"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated From 831d9e693e1d9b57d8a05e7eca807a309604e932 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 21 Apr 2024 01:02:20 -0400 Subject: [PATCH 02/17] fix OSError on win Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 62f4679986..6938d04d05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,7 +199,7 @@ test-command = [ "pytest {project}/source/tests/tf/test_lammps.py" ] test-extras = ["cpu", "test", "lmp", "ipi"] -before-test = ["pip install replace-pip-with-uv-pip"] +before-test = ["python -m pip install replace-pip-with-uv-pip"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated From 3ff8039c194c5b932e94963a26ca458a38fbb09f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 29 Apr 2024 22:04:15 -0400 Subject: [PATCH 03/17] pin aarch64 h5py to <3.11 Signed-off-by: Jinzhe Zeng --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6938d04d05..5a86fd3288 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,10 @@ test-command = [ "pytest {project}/source/tests/tf/test_lammps.py" ] test-extras = ["cpu", "test", "lmp", "ipi"] +# TODO: build(wheel): unpin h5py on aarch64 +# Revert after https://github.com/h5py/h5py/issues/2408 is fixed; +# or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. +test-requires = ["h5py<3.11; platform_machine=='aarch64'"] before-test = ["python -m pip install replace-pip-with-uv-pip"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] From 44947ea83b5d090e7b12d23b36f396cba9ab8b42 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 29 Apr 2024 22:21:14 -0400 Subject: [PATCH 04/17] remove space --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a86fd3288..017c1646f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,7 +202,7 @@ test-extras = ["cpu", "test", "lmp", "ipi"] # TODO: build(wheel): unpin h5py on aarch64 # Revert after https://github.com/h5py/h5py/issues/2408 is fixed; # or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. -test-requires = ["h5py<3.11; platform_machine=='aarch64'"] +test-requires = ["h5py<3.11;platform_machine=='aarch64'"] before-test = ["python -m pip install replace-pip-with-uv-pip"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] From f87e4e2582548252fbebaea798a163404df171bb Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 17:58:38 -0400 Subject: [PATCH 05/17] use build[uv] Signed-off-by: Jinzhe Zeng --- .github/workflows/build_wheel.yml | 23 +++++++---------------- .gitignore | 1 + pyproject.toml | 3 ++- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 18fd7a1ac1..0f6db978d5 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -74,23 +74,19 @@ jobs: with: # https://github.com/pypa/setuptools_scm/issues/480 fetch-depth: 0 + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + if: runner.os != 'Linux' - uses: docker/setup-qemu-action@v3 name: Setup QEMU if: matrix.platform_id == 'manylinux_aarch64' && matrix.os == 'ubuntu-latest' # detect version in advance. See #3168 - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.11' - cache: 'pip' - if: matrix.dp_pkg_name == 'deepmd-kit-cu11' - run: | - python -m pip install setuptools_scm - python -c "from setuptools_scm import get_version;print('SETUPTOOLS_SCM_PRETEND_VERSION='+get_version())" >> $GITHUB_ENV + echo "SETUPTOOLS_SCM_PRETEND_VERSION=$(pipx run uv tool run --from setuptools_scm python -m setuptools_scm)" >> $GITHUB_ENV rm -rf .git if: matrix.dp_pkg_name == 'deepmd-kit-cu11' - name: Build wheels - uses: pypa/cibuildwheel@v2.17 + uses: pypa/cibuildwheel@v2.19 env: CIBW_BUILD_VERBOSITY: 1 CIBW_ARCHS: all @@ -98,6 +94,7 @@ jobs: DP_VARIANT: ${{ matrix.dp_variant }} CUDA_VERSION: ${{ matrix.cuda_version }} DP_PKG_NAME: ${{ matrix.dp_pkg_name }} + CIBW_BUILD_FRONTEND: 'build[uv]' - uses: actions/upload-artifact@v4 with: name: cibw-cp${{ matrix.python }}-${{ matrix.platform_id }}-cu${{ matrix.cuda_version }}-${{ strategy.job-index }} @@ -109,14 +106,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.11' - cache: 'pip' - - run: python -m pip install build - name: Build sdist - run: python -m build --sdist + run: pipx run uv tool run --with build[uv] --from build python -m build --installer uv - uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index 5e30cf3167..c531a76177 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ build_cc_tests build_c_tests build_c/ libdeepmd_c/ +.uv/ diff --git a/pyproject.toml b/pyproject.toml index 017c1646f5..afedba0783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -203,7 +203,6 @@ test-extras = ["cpu", "test", "lmp", "ipi"] # Revert after https://github.com/h5py/h5py/issues/2408 is fixed; # or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. test-requires = ["h5py<3.11;platform_machine=='aarch64'"] -before-test = ["python -m pip install replace-pip-with-uv-pip"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated @@ -234,6 +233,8 @@ before-all = [ """rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux""", """{ if [ "$(uname -m)" = "x86_64" ] ; then yum config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && yum install -y cuda-nvcc-${CUDA_VERSION/./-} cuda-cudart-devel-${CUDA_VERSION/./-}; fi }""", "yum install -y mpich-devel", + # uv is not available in the old manylinux image + "pipx install uv", ] [tool.cibuildwheel.windows] From bc05f4dc357b510a4cb86405d3b1f8c003fe3cbf Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:00:50 -0400 Subject: [PATCH 06/17] skip h5py 3.11.0 for linux aarch64 Signed-off-by: Jinzhe Zeng --- backend/find_tensorflow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 6d13f7fee5..4e1e491492 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -156,11 +156,15 @@ def get_tf_requirement(tf_version: str = "") -> dict: "tensorflow; platform_machine=='aarch64' or (platform_machine=='arm64' and platform_system == 'Darwin')", # https://github.com/tensorflow/tensorflow/issues/61830 "tensorflow-cpu!=2.15.*; platform_system=='Windows'", + # https://github.com/h5py/h5py/issues/2408 + "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], "gpu": [ "tensorflow", "tensorflow-metal; platform_machine=='arm64' and platform_system == 'Darwin'", + # https://github.com/h5py/h5py/issues/2408 + "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], **extra_select, From bd20c9722a293265f27669494580d7ba6ca97429 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:19:46 -0400 Subject: [PATCH 07/17] remove test_requires Signed-off-by: Jinzhe Zeng --- backend/find_tensorflow.py | 6 ++++-- pyproject.toml | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 4e1e491492..e080464fb4 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -156,14 +156,16 @@ def get_tf_requirement(tf_version: str = "") -> dict: "tensorflow; platform_machine=='aarch64' or (platform_machine=='arm64' and platform_system == 'Darwin')", # https://github.com/tensorflow/tensorflow/issues/61830 "tensorflow-cpu!=2.15.*; platform_system=='Windows'", - # https://github.com/h5py/h5py/issues/2408 + # TODO: build(wheel): unpin h5py on aarch64 + # Revert after https://github.com/h5py/h5py/issues/2408 is fixed; + # or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], "gpu": [ "tensorflow", "tensorflow-metal; platform_machine=='arm64' and platform_system == 'Darwin'", - # https://github.com/h5py/h5py/issues/2408 + # See above. "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], diff --git a/pyproject.toml b/pyproject.toml index a0edcb87cf..9601e350d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,10 +202,6 @@ test-command = [ "pytest {project}/source/tests/tf/test_lammps.py" ] test-extras = ["cpu", "test", "lmp", "ipi"] -# TODO: build(wheel): unpin h5py on aarch64 -# Revert after https://github.com/h5py/h5py/issues/2408 is fixed; -# or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. -test-requires = ["h5py<3.11;platform_machine=='aarch64'"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated From bae8bb467555d50d371be8d4bbab098cbe8509d3 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:22:19 -0400 Subject: [PATCH 08/17] requires h5py >=3.6 Signed-off-by: Jinzhe Zeng --- backend/find_tensorflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index e080464fb4..8ba62c9814 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -159,14 +159,15 @@ def get_tf_requirement(tf_version: str = "") -> dict: # TODO: build(wheel): unpin h5py on aarch64 # Revert after https://github.com/h5py/h5py/issues/2408 is fixed; # or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. - "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", + # 3.6.0 is the first version to have aarch64 wheels. + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], "gpu": [ "tensorflow", "tensorflow-metal; platform_machine=='arm64' and platform_system == 'Darwin'", # See above. - "h5py<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], **extra_select, From 1e4df127a969e45d2cc5a932ed2eb86673adce0d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:24:01 -0400 Subject: [PATCH 09/17] upgrade build Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9601e350d4..0ff0a796e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,8 @@ before-all = [ "yum install -y mpich-devel", # uv is not available in the old manylinux image "pipx install uv", + # old build doesn't support uv + "pipx upgrade build", ] [tool.cibuildwheel.windows] From 2c462148ca0ce4e0a0a936e73213e031b3c5402a Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:39:48 -0400 Subject: [PATCH 10/17] fix upgrading build Signed-off-by: Jinzhe Zeng --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ff0a796e4..003729f076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,8 +234,10 @@ before-all = [ "yum install -y mpich-devel", # uv is not available in the old manylinux image "pipx install uv", +] +before-build = [ # old build doesn't support uv - "pipx upgrade build", + "uv pip install --system -U build", ] [tool.cibuildwheel.windows] From 6e87c1e1eddc7a38549f24ba10231ad828b5166f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:42:31 -0400 Subject: [PATCH 11/17] only upgrade uv in x86_64 Signed-off-by: Jinzhe Zeng --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 003729f076..c4b9542cb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,11 +233,11 @@ before-all = [ """{ if [ "$(uname -m)" = "x86_64" ] ; then yum config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && yum install -y cuda-nvcc-${CUDA_VERSION/./-} cuda-cudart-devel-${CUDA_VERSION/./-}; fi }""", "yum install -y mpich-devel", # uv is not available in the old manylinux image - "pipx install uv", + """{ if [ "$(uname -m)" = "x86_64" ] ; then pipx install uv; fi }""", ] before-build = [ # old build doesn't support uv - "uv pip install --system -U build", + """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] [tool.cibuildwheel.windows] From a4547f7cfed18911e2794bc770b8733226524c0b Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 18:51:37 -0400 Subject: [PATCH 12/17] build sdist only in the sdist step Signed-off-by: Jinzhe Zeng --- .github/workflows/build_wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 0f6db978d5..7fada27493 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -107,7 +107,7 @@ jobs: with: fetch-depth: 0 - name: Build sdist - run: pipx run uv tool run --with build[uv] --from build python -m build --installer uv + run: pipx run uv tool run --with build[uv] --from build python -m build --installer uv --sdist - uses: actions/upload-artifact@v4 with: From c07a84ca7dddbc747ec98fed8bbcdb043c7ea090 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 19:06:09 -0400 Subject: [PATCH 13/17] for unclear reason, still need test-requires Signed-off-by: Jinzhe Zeng --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c4b9542cb1..cb3fb7c09f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,9 @@ before-build = [ # old build doesn't support uv """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] +test-requires = [ + """h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'""", +] [tool.cibuildwheel.windows] environment = { PIP_PREFER_BINARY="1" } From 934311372979964e278d30b92ae5d10ce8697fc6 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 19:34:11 -0400 Subject: [PATCH 14/17] add quotes --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cb3fb7c09f..b0e0e5d8f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,7 +240,7 @@ before-build = [ """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] test-requires = [ - """h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'""", + """ "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'" """, ] [tool.cibuildwheel.windows] From acba73aae4f3a5ec7aa699cb4dd046591e1b505c Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 19:52:13 -0400 Subject: [PATCH 15/17] Revert "add quotes" This reverts commit 934311372979964e278d30b92ae5d10ce8697fc6. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b0e0e5d8f5..cb3fb7c09f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,7 +240,7 @@ before-build = [ """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] test-requires = [ - """ "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'" """, + """h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'""", ] [tool.cibuildwheel.windows] From de3d06c46a8276d8a7c8348c6c3354fe70a2d1c3 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 19:52:15 -0400 Subject: [PATCH 16/17] Revert "for unclear reason, still need test-requires" This reverts commit c07a84ca7dddbc747ec98fed8bbcdb043c7ea090. --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cb3fb7c09f..c4b9542cb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,9 +239,6 @@ before-build = [ # old build doesn't support uv """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] -test-requires = [ - """h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'""", -] [tool.cibuildwheel.windows] environment = { PIP_PREFER_BINARY="1" } From c14325e01e6569ddefb89494b2fdb0b763445dff Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 19:53:14 -0400 Subject: [PATCH 17/17] add to dependencies Signed-off-by: Jinzhe Zeng --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c4b9542cb1..ebb60b1712 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ 'typing_extensions; python_version < "3.8"', 'importlib_metadata>=1.4; python_version < "3.8"', 'h5py', + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", 'wcmatch', 'packaging', 'ml_dtypes',