diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index 56c6002aa..e1714d011 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -36,7 +36,7 @@ "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:33") + "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:34") }, "aarch64": { "provider": "aws", @@ -101,7 +101,7 @@ def main(args): "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:16" + "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:17" }, "commands": [ ".buildkite/scripts/steps/build_and_test.sh" diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py index 681da7e13..4b72b6964 100755 --- a/.buildkite/pipelines/build_macos.json.py +++ b/.buildkite/pipelines/build_macos.json.py @@ -33,7 +33,7 @@ agents = { "aarch64": { "provider": "orka", - "image": "ml-macos-13-arm-004.orkasi" + "image": "ml-macos-13-arm-005.orkasi" } } envs = { diff --git a/.buildkite/pipelines/build_windows.json.py b/.buildkite/pipelines/build_windows.json.py index b357944f8..4872adcd2 100755 --- a/.buildkite/pipelines/build_windows.json.py +++ b/.buildkite/pipelines/build_windows.json.py @@ -45,7 +45,7 @@ def main(args): "provider": "gcp", "machineType": "c2-standard-16", "minCpuPlatform": "Intel Cascade Lake", - "image": "family/ml-cpp-4-windows-2022", + "image": "family/ml-cpp-5-windows-2022", }, "commands": [ f'if ( "{args.action}" -eq "debug" ) {{\$Env:ML_DEBUG="1"}}', diff --git a/.ci/orka/README.md b/.ci/orka/README.md index a85520183..6b9d39c36 100644 --- a/.ci/orka/README.md +++ b/.ci/orka/README.md @@ -9,7 +9,7 @@ ## Files - `install.sh` The script that does the software installs on the image -- `orka-macos-12-arm.pkr.hcl` The packer definition for a MacOS 12 ARM builder image +- `orka-macos-13-arm.pkr.hcl` The packer definition for a MacOS 13 ARM builder image ## Set Up Packer @@ -26,7 +26,7 @@ Make sure you are connected to the Orka VM. Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-prod.elastic.dev and VAULT_TOKEN must be set appropriately in the environment. -Run the following to create the image (MacOS 12 ARM in this example): +Run the following to create the image (MacOS 13 ARM in this example): ``` packer build orka-macos-13-arm.pkr.hcl @@ -63,7 +63,7 @@ The ML packer scripts do the following: * Install JDK `11.0.25` * Install CMake `3.30.5` * Install Boost `1.86.0` from source - * Install PyTorch `2.5.1` from source + * Install PyTorch `2.7.1` from source ## Caveats diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl index bfe9dc96b..5ec0880f1 100644 --- a/.ci/orka/orka-macos-13-arm.pkr.hcl +++ b/.ci/orka/orka-macos-13-arm.pkr.hcl @@ -18,7 +18,7 @@ locals { source "macstadium-orka" "image" { source_image = "generic-13-ventura-arm-002.orkasi" - image_name = "ml-macos-13-arm-004.orkasi" + image_name = "ml-macos-13-arm-005.orkasi" orka_endpoint = local.orka_endpoint orka_user = local.orka_user orka_password = local.orka_password diff --git a/.ci/orka/third_party_deps.sh b/.ci/orka/third_party_deps.sh index 585edc511..b48bef31a 100644 --- a/.ci/orka/third_party_deps.sh +++ b/.ci/orka/third_party_deps.sh @@ -34,7 +34,7 @@ sudo rm -rf boost_1_86_0 sudo pip3 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses # Build and install PyTorch -git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ +git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -59,7 +59,7 @@ export DNNL_TARGET_ARCH=AARCH64 export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 python3 setup.py install diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv index 6750bd122..04bd473a0 100644 --- a/3rd_party/licenses/pytorch-INFO.csv +++ b/3rd_party/licenses/pytorch-INFO.csv @@ -1,2 +1,2 @@ name,version,revision,url,license,copyright,sourceURL -PyTorch,2.5.1,a8d6afb511a69687bbb2b7e88a3cf67917e1697e,https://pytorch.org,BSD-3-Clause,, +PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,, diff --git a/build-setup/linux.md b/build-setup/linux.md index 0621c2121..0fc99dffa 100644 --- a/build-setup/linux.md +++ b/build-setup/linux.md @@ -333,7 +333,7 @@ Then copy the shared libraries to the system directory: (cd /opt/intel/oneapi/mkl/2024.0 && tar cf - lib) | (cd /usr/local/gcc133 && sudo tar xvf -) ``` -### PyTorch 2.5.1 +### PyTorch 2.7.1 (This step requires a reasonable amount of memory. It failed on a machine with 8GB of RAM. It succeeded on a 16GB machine. You can specify the number of parallel jobs using environment variable MAX_JOBS. Lower number of jobs will reduce memory usage.) @@ -352,7 +352,7 @@ sudo /usr/local/gcc133/bin/python3.12 -m pip install numpy pyyaml setuptools cff Then obtain the PyTorch code: ``` -git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git +git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git cd pytorch git submodule sync git submodule update --init --recursive @@ -386,7 +386,7 @@ export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 /usr/local/gcc133/bin/python3.12 setup.py install ``` diff --git a/build-setup/macos.md b/build-setup/macos.md index 2ca1765ac..f34257ef6 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -133,18 +133,18 @@ Download the graphical installer for Python 3.10.10 from diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile index f485deca2..004892a09 100644 --- a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile @@ -27,7 +27,7 @@ RUN dnf -y update && \ RUN \ mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \ cd /usr/local/sysroot-aarch64-linux-gnu/usr && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-16.tar.bz2 | tar jxf - && \ + wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-17.tar.bz2 | tar jxf - && \ cd .. && \ ln -s usr/lib lib && \ ln -s usr/lib64 lib64 diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile index e4d03181d..1524b0045 100644 --- a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:16 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17 MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile index f3cb5d19c..7e8bd99c0 100644 --- a/dev-tools/docker/linux_aarch64_native_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_image/Dockerfile @@ -169,7 +169,7 @@ RUN \ # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -183,7 +183,7 @@ RUN \ export USE_MKLDNN=ON && \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install && \ diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile index aa3c6c33c..008376518 100644 --- a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:16 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17 MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile index 587229e07..ad5ab9483 100644 --- a/dev-tools/docker/linux_builder/Dockerfile +++ b/dev-tools/docker/linux_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 +FROM docker.elastic.co/ml-dev/ml-linux-build:34 LABEL maintainer="Valeriy Khakhutskyy " diff --git a/dev-tools/docker/linux_dependency_builder_image/Dockerfile b/dev-tools/docker/linux_dependency_builder_image/Dockerfile index bab0880c0..1f172c33b 100644 --- a/dev-tools/docker/linux_dependency_builder_image/Dockerfile +++ b/dev-tools/docker/linux_dependency_builder_image/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 AS builder +FROM docker.elastic.co/ml-dev/ml-linux-build:34 AS builder # This is basically automating the setup instructions in build-setup/linux.md diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile index 34426f99e..7099e1a6d 100644 --- a/dev-tools/docker/linux_image/Dockerfile +++ b/dev-tools/docker/linux_image/Dockerfile @@ -183,7 +183,7 @@ gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS. # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -198,7 +198,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install && \ diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile index 058709cae..7e3ef0ba5 100644 --- a/dev-tools/docker/linux_tester/Dockerfile +++ b/dev-tools/docker/linux_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 +FROM docker.elastic.co/ml-dev/ml-linux-build:34 LABEL maintainer="Valeriy Khakhutskyy " diff --git a/dev-tools/docker/ubuntu_devbox/Dockerfile b/dev-tools/docker/ubuntu_devbox/Dockerfile index 4da7076a9..0d5a7becb 100644 --- a/dev-tools/docker/ubuntu_devbox/Dockerfile +++ b/dev-tools/docker/ubuntu_devbox/Dockerfile @@ -165,7 +165,7 @@ ENV PATH=/usr/local/gcc133/bin:/usr/bin:/bin:/usr/sbin:/sbin # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -179,7 +179,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index fc1eac34a..5edc131f6 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -80,7 +80,7 @@ do TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ prefetch_docker_base_image "$DOCKERFILE" - docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . + docker build --no-cache --force-rm -t $TEMP_TAG --progress=plain --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . # Using tar to copy the build artifacts out of the container seems more reliable # than docker cp, and also means the files end up with the correct uid/gid docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "tar cf - build/distributions && sleep 30" | tar xvf - diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index aed18fa60..91b9e3a5b 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -88,7 +88,7 @@ do TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ prefetch_docker_base_image "$DOCKERFILE" - docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg BOOST_TEST_OUTPUT_FORMAT_FLAGS=$BOOST_TEST_OUTPUT_FORMAT_FLAGS --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . + docker build --no-cache --force-rm -t $TEMP_TAG --progress=plain --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg BOOST_TEST_OUTPUT_FORMAT_FLAGS=$BOOST_TEST_OUTPUT_FORMAT_FLAGS --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . # Using tar to copy the build and test artifacts out of the container seems # more reliable than docker cp, and also means the files end up with the # correct uid/gid diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 3b119797c..f4510b40c 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -27,12 +27,17 @@ //=== Bug Fixes //=== Regressions +== {es} version 9.2.0 + +=== Enhancements == {es} version 9.1.0 === Enhancements * Track memory used in the hierarchical results normalizer. (See {ml-pull}2831[#2831].) +* Update the PyTorch library to version 2.7.1. (See {ml-pull}2879[#2879].) + === Bug Fixes diff --git a/include/maths/common/CXMeans.h b/include/maths/common/CXMeans.h index a8b1782af..c3e888cfe 100644 --- a/include/maths/common/CXMeans.h +++ b/include/maths/common/CXMeans.h @@ -26,6 +26,7 @@ #include #include #include +#include #include namespace ml { @@ -104,6 +105,15 @@ class CXMeans { //! Get the cluster checksum. std::uint64_t checksum() const { return m_Checksum; } + //! \brief Stream insertion operator for CCluster. + //! This is defined here for use in Boost unit tests to print clusters + //! when a test fails. + friend std::ostream& operator<<(std::ostream& strm, const CCluster& cluster) { + return strm << "Cluster{ cost: " << cluster.cost() + << ", centre: " << cluster.centre().toDelimited() + << ", points checksum: " << cluster.checksum() << " }"; + } + private: //! The information criterion cost of this cluster. double m_Cost; diff --git a/lib/maths/common/unittest/CXMeansTest.cc b/lib/maths/common/unittest/CXMeansTest.cc index 8accfd23e..756d349f5 100644 --- a/lib/maths/common/unittest/CXMeansTest.cc +++ b/lib/maths/common/unittest/CXMeansTest.cc @@ -32,41 +32,6 @@ using TVector2 = ml::maths::common::CVectorNx1; using TVector4 = ml::maths::common::CVectorNx1; using TVector4Vec = std::vector; -// TODO boost test: replace these output operators -// with a more generic way of printing clusters in -// the production code -namespace ml { -namespace maths { -namespace common { - -std::ostream& operator<<(std::ostream& strm, - const typename CXMeans::CCluster& cluster) { - return strm << "Cluster{ cost: " << cluster.cost() - << ", centre: " << cluster.centre().toDelimited() - << ", points checksum: " << cluster.checksum() << " }"; -} - -std::ostream& operator<<(std::ostream& strm, - const typename CXMeans::CCluster& cluster) { - return strm << "Cluster{ cost: " << cluster.cost() - << ", centre: " << cluster.centre().toDelimited() - << ", points checksum: " << cluster.checksum() << " }"; -} - -std::ostream& operator<<(std::ostream& strm, - const typename CXMeans::CCluster& cluster) { - strm << "Cluster{ cost: " << cluster.cost() << ", centre: "; - const char* delim{"("}; - for (const auto& element : cluster.centre()) { - strm << delim << element.toDelimited() << ')'; - delim = ", ("; - } - return strm << ", points checksum: " << cluster.checksum() << " }"; -} -} -} -} - BOOST_AUTO_TEST_SUITE(CXMeansTest) using namespace ml;