From d9f67f073575f5abb2d21070bc7df9ceb2427dc4 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 17 Nov 2025 22:13:50 +0000 Subject: [PATCH] [ML] Bump PyTorch version to 2.7.1 PyTorch 2.7.1 contains essential security fixes. We need to upgrade our build images accordingly. This initial commit contains the changes for the Linux build images. macOS and Windows images willbe updated in due course. This is essentially a backport of #2863 to the 8.19 branch. --- .buildkite/pipelines/build_linux.json.py | 4 ++-- 3rd_party/licenses/pytorch-INFO.csv | 2 +- build-setup/linux.md | 6 +++--- build-setup/macos.md | 6 +++--- build-setup/windows.md | 6 +++--- dev-tools/docker/README.md | 8 ++++---- dev-tools/docker/build_linux_aarch64_cross_build_image.sh | 2 +- .../docker/build_linux_aarch64_native_build_image.sh | 2 +- dev-tools/docker/build_linux_build_image.sh | 2 +- dev-tools/docker/linux_aarch64_cross_builder/Dockerfile | 2 +- dev-tools/docker/linux_aarch64_cross_image/Dockerfile | 2 +- dev-tools/docker/linux_aarch64_native_builder/Dockerfile | 2 +- dev-tools/docker/linux_aarch64_native_image/Dockerfile | 6 +++--- dev-tools/docker/linux_aarch64_native_tester/Dockerfile | 2 +- dev-tools/docker/linux_builder/Dockerfile | 2 +- .../docker/linux_dependency_builder_image/Dockerfile | 2 +- dev-tools/docker/linux_image/Dockerfile | 6 +++--- dev-tools/docker/linux_tester/Dockerfile | 2 +- dev-tools/download_macos_deps.sh | 2 +- dev-tools/download_windows_deps.ps1 | 6 +++--- docs/CHANGELOG.asciidoc | 6 ++++++ 21 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index 3178391fb..c9443a17d 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -36,7 +36,7 @@ "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:32") + "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:32a") }, "aarch64": { "provider": "aws", @@ -101,7 +101,7 @@ def main(args): "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:15" + "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:15a" }, "commands": [ ".buildkite/scripts/steps/build_and_test.sh" diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv index 6750bd122..04bd473a0 100644 --- a/3rd_party/licenses/pytorch-INFO.csv +++ b/3rd_party/licenses/pytorch-INFO.csv @@ -1,2 +1,2 @@ name,version,revision,url,license,copyright,sourceURL -PyTorch,2.5.1,a8d6afb511a69687bbb2b7e88a3cf67917e1697e,https://pytorch.org,BSD-3-Clause,, +PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,, diff --git a/build-setup/linux.md b/build-setup/linux.md index adcc5d1a0..22c241a6e 100644 --- a/build-setup/linux.md +++ b/build-setup/linux.md @@ -329,7 +329,7 @@ Then copy the shared libraries to the system directory: (cd /opt/intel/oneapi/mkl/2024.0 && tar cf - lib) | (cd /usr/local/gcc103 && sudo tar xvf -) ``` -### PyTorch 2.5.1 +### PyTorch 2.7.1 (This step requires a reasonable amount of memory. It failed on a machine with 8GB of RAM. It succeeded on a 16GB machine. You can specify the number of parallel jobs using environment variable MAX_JOBS. Lower number of jobs will reduce memory usage.) @@ -348,7 +348,7 @@ sudo /usr/local/gcc103/bin/python3.10 -m pip install install numpy pyyaml setupt Then obtain the PyTorch code: ``` -git clone --depth=1 --branch=v2.5.1 git@github.com:pytorch/pytorch.git +git clone --depth=1 --branch=v2.7.1 git@github.com:pytorch/pytorch.git cd pytorch git submodule sync git submodule update --init --recursive @@ -383,7 +383,7 @@ export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 /usr/local/gcc103/bin/python3.10 setup.py install ``` diff --git a/build-setup/macos.md b/build-setup/macos.md index 6f6b2afae..b2441b9ae 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -127,7 +127,7 @@ Download the graphical installer for Python 3.10.9 from diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile index 20559d1bb..4a8944f25 100644 --- a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile @@ -27,7 +27,7 @@ RUN \ RUN \ mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \ cd /usr/local/sysroot-aarch64-linux-gnu/usr && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-15.tar.bz2 | tar jxf - && \ + wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-15a.tar.bz2 | tar jxf - && \ cd .. && \ ln -s usr/lib lib && \ ln -s usr/lib64 lib64 diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile index ec326a475..97a960a30 100644 --- a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile index 7f533f224..6dcb50696 100644 --- a/dev-tools/docker/linux_aarch64_native_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_image/Dockerfile @@ -35,7 +35,7 @@ ARG build_dir=/usr/src # Build gcc 10.3 RUN \ cd ${build_dir} && \ - wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ + wget --quiet -O - https://mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ cd gcc-10.3.0 && \ contrib/download_prerequisites && \ sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ @@ -141,7 +141,7 @@ RUN \ # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -155,7 +155,7 @@ RUN \ export USE_MKLDNN=ON && \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ /usr/local/bin/python3.10 setup.py install && \ mkdir /usr/local/gcc103/include/pytorch && \ diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile index 3cccfbfc3..eb3d55804 100644 --- a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:15a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile index f9f2ecdc0..d484c80b5 100644 --- a/dev-tools/docker/linux_builder/Dockerfile +++ b/dev-tools/docker/linux_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 +FROM docker.elastic.co/ml-dev/ml-linux-build:32a MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_dependency_builder_image/Dockerfile b/dev-tools/docker/linux_dependency_builder_image/Dockerfile index 33defeecc..9d2819c2f 100644 --- a/dev-tools/docker/linux_dependency_builder_image/Dockerfile +++ b/dev-tools/docker/linux_dependency_builder_image/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 AS builder +FROM docker.elastic.co/ml-dev/ml-linux-build:32a AS builder # This is basically automating the setup instructions in build-setup/linux.md diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile index 65ba95877..4f876b4ad 100644 --- a/dev-tools/docker/linux_image/Dockerfile +++ b/dev-tools/docker/linux_image/Dockerfile @@ -35,7 +35,7 @@ ARG build_dir=/usr/src # Build gcc 10.3 RUN \ cd ${build_dir} && \ - wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ + wget --quiet -O - https://mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.3.0/gcc-10.3.0.tar.gz | tar zxf - && \ cd gcc-10.3.0 && \ contrib/download_prerequisites && \ sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ @@ -154,7 +154,7 @@ gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS. # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -169,7 +169,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.10 setup.py install && \ diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile index 8c7f6c6eb..d4e7845e6 100644 --- a/dev-tools/docker/linux_tester/Dockerfile +++ b/dev-tools/docker/linux_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:32 +FROM docker.elastic.co/ml-dev/ml-linux-build:32a MAINTAINER David Roberts diff --git a/dev-tools/download_macos_deps.sh b/dev-tools/download_macos_deps.sh index 0b7294fda..d4c8f3153 100755 --- a/dev-tools/download_macos_deps.sh +++ b/dev-tools/download_macos_deps.sh @@ -28,7 +28,7 @@ case `uname -m` in ARCHIVE=local-x86_64-apple-macosx12.0-1.tar.bz2 ;; arm64) - ARCHIVE=local-arm64-apple-macosx11.1-12.tar.bz2 + ARCHIVE=local-arm64-apple-macosx12.2-2.tar.bz2 ;; *) diff --git a/dev-tools/download_windows_deps.ps1 b/dev-tools/download_windows_deps.ps1 index dbda28994..08652c94d 100755 --- a/dev-tools/download_windows_deps.ps1 +++ b/dev-tools/download_windows_deps.ps1 @@ -9,11 +9,11 @@ # limitation. # $ErrorActionPreference="Stop" -$Archive="usr-x86_64-windows-2016-15.zip" +$Archive="usr-x86_64-windows-2016-16.zip" $Destination="C:\" -# If PyTorch is not version 2.5.1 then we need the latest download +# If PyTorch is not version 2.7.1 then we need the latest download if (!(Test-Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h") -Or - !(Select-String -Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h" -Pattern "2.5.1" -Quiet)) { + !(Select-String -Path "$Destination\usr\local\include\pytorch\torch\csrc\api\include\torch\version.h" -Pattern "2.7.1" -Quiet)) { Remove-Item "$Destination\usr" -Recurse -Force -ErrorAction Ignore $ZipSource="https://storage.googleapis.com/elastic-ml-public/dependencies/$Archive" $ZipDestination="$Env:TEMP\$Archive" diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index a77ed44de..80dea17b7 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -28,6 +28,12 @@ //=== Regressions +== {es} version 8.19.8 + +=== Enhancements + +* Update the PyTorch library to version 2.7.1. (See {ml-pull}2880[#2880].) + == {es} version 8.19.0 === Enhancements