From 3e6b8654b0f5cec745f2260faac9bd3e4af0669f Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 11:46:27 +0800 Subject: [PATCH 01/21] skip elpa compiling in Dockerfile.intel --- Dockerfile.intel | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index f4e2a042ff2..38e391cd3e7 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -24,17 +24,17 @@ SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html -RUN source /opt/intel/oneapi/setvars.sh && \ - cd /tmp && \ - ELPA_VER=2022.11.001 && \ - wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ - tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ - cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ - make -j`nproc` && \ - make PREFIX=/usr/local install && \ - ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ - cd /tmp && rm -rf elpa-$ELPA_VER +#RUN source /opt/intel/oneapi/setvars.sh && \ +# cd /tmp && \ +# ELPA_VER=2022.11.001 && \ +# wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ +# tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ +# cd elpa-$ELPA_VER && mkdir build && cd build && \ +# ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ +# make -j`nproc` && \ +# make PREFIX=/usr/local install && \ +# ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ +# cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ --no-check-certificate --quiet -O libtorch.zip && \ @@ -47,7 +47,7 @@ ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/deve RUN source /opt/intel/oneapi/setvars.sh && \ git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ cd abacus-develop && \ - cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON && \ + cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DUSE_ELPA=0 && \ cmake --build build -j`nproc` && \ cmake --install build && \ rm -rf build && \ From fc64be9d347109325a9d0872e112ca4e5b15bb0d Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 15:44:14 +0800 Subject: [PATCH 02/21] Fix Dockerfile.intel --- Dockerfile.intel | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 38e391cd3e7..445fe3ef275 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -12,29 +12,24 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO # To save disk space, only install the essential components, but not the whole toolkit. RUN apt-get update && \ - apt-get install -y \ - intel-oneapi-compiler-dpcpp-cpp \ - intel-oneapi-compiler-fortran \ - intel-oneapi-mkl-devel \ - intel-oneapi-mpi-devel \ - intel-oneapi-vtune + sudo apt -y install intel-basekit intel-hpckit RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html -#RUN source /opt/intel/oneapi/setvars.sh && \ -# cd /tmp && \ -# ELPA_VER=2022.11.001 && \ -# wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ -# tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ -# cd elpa-$ELPA_VER && mkdir build && cd build && \ -# ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ -# make -j`nproc` && \ -# make PREFIX=/usr/local install && \ -# ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ -# cd /tmp && rm -rf elpa-$ELPA_VER +RUN source /opt/intel/oneapi/setvars.sh && \ + cd /tmp && \ + ELPA_VER=2022.11.001 && \ + wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ + tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ + cd elpa-$ELPA_VER && mkdir build && cd build && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ + make -j`nproc` && \ + make PREFIX=/usr/local install && \ + cp -r /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ + cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ --no-check-certificate --quiet -O libtorch.zip && \ @@ -47,7 +42,7 @@ ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/deve RUN source /opt/intel/oneapi/setvars.sh && \ git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ cd abacus-develop && \ - cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DUSE_ELPA=0 && \ + cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON && \ cmake --build build -j`nproc` && \ cmake --install build && \ rm -rf build && \ From 38428a4d0b96ef4361211e7904c0f8491d84b67b Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 18:17:27 +0800 Subject: [PATCH 03/21] try avx512 --- Dockerfile.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 445fe3ef275..23981a0c943 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -16,7 +16,7 @@ RUN apt-get update && \ RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc SHELL ["/bin/bash", "-c"] -ENV CC=mpiicx CXX=mpiicpx FC=mpiifx +ENV CC=mpiicx CXX=mpiicpx FC=mpiifx CFLAGS="-mavx512f -mavx512dq -mavx512cd -mavx512bw -mavx512vl" # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html RUN source /opt/intel/oneapi/setvars.sh && \ From cdb4e499f0b4dbf50c3737b92bd05eeb32607e0c Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 18:20:22 +0800 Subject: [PATCH 04/21] try avx512 for fortran compiler --- Dockerfile.intel | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.intel b/Dockerfile.intel index 23981a0c943..69a48a539c4 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -22,6 +22,7 @@ ENV CC=mpiicx CXX=mpiicpx FC=mpiifx CFLAGS="-mavx512f -mavx512dq -mavx512cd -mav RUN source /opt/intel/oneapi/setvars.sh && \ cd /tmp && \ ELPA_VER=2022.11.001 && \ + FCFLAGS="-mavx512f -mavx512dq -mavx512cd -mavx512bw -mavx512vl" && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ From 5334337bb6c6e6f96cbdcda23db17e188d20807a Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 19:52:39 +0800 Subject: [PATCH 05/21] try avx512 --- Dockerfile.intel | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 69a48a539c4..99a54704f94 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -16,13 +16,12 @@ RUN apt-get update && \ RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc SHELL ["/bin/bash", "-c"] -ENV CC=mpiicx CXX=mpiicpx FC=mpiifx CFLAGS="-mavx512f -mavx512dq -mavx512cd -mavx512bw -mavx512vl" +ENV CC=mpiicx CXX=mpiicpx FC=mpiifx CFLAGS="-mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -msse4a -msha -maes -mavx -mfma -mavx2" # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html RUN source /opt/intel/oneapi/setvars.sh && \ cd /tmp && \ ELPA_VER=2022.11.001 && \ - FCFLAGS="-mavx512f -mavx512dq -mavx512cd -mavx512bw -mavx512vl" && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ From dd34622357600eac92a4668e8d591a96c4201dcc Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sat, 18 Nov 2023 20:22:09 +0800 Subject: [PATCH 06/21] tryagain --- Dockerfile.intel | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 99a54704f94..da5db0b2afe 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -16,7 +16,7 @@ RUN apt-get update && \ RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc SHELL ["/bin/bash", "-c"] -ENV CC=mpiicx CXX=mpiicpx FC=mpiifx CFLAGS="-mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -msse4a -msha -maes -mavx -mfma -mavx2" +ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html RUN source /opt/intel/oneapi/setvars.sh && \ @@ -25,7 +25,8 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ + CFLAGS="-mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -msse4a -msha -maes -mavx -mfma -mavx2" && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --without-threading-support-check-during-build && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ cp -r /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ From ec287c2735c54d5ba4e0e9fe721077246fef5699 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sun, 19 Nov 2023 18:26:14 +0800 Subject: [PATCH 07/21] try old version elpa --- Dockerfile.intel | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index da5db0b2afe..01b4d4a887e 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -21,15 +21,14 @@ ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html RUN source /opt/intel/oneapi/setvars.sh && \ cd /tmp && \ - ELPA_VER=2022.11.001 && \ + ELPA_VER=2021.05.002 && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - CFLAGS="-mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -msse4a -msha -maes -mavx -mfma -mavx2" && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --without-threading-support-check-during-build && \ + ../configure FCFLAGS="-qmkl=cluster" --enable-openmp && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ - cp -r /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ + ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ From fb93935f06833430592dd5c8a47aa996dc9a5df3 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sun, 19 Nov 2023 18:29:26 +0800 Subject: [PATCH 08/21] fix ln to cp --- Dockerfile.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 01b4d4a887e..3fba439dd64 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -28,7 +28,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ ../configure FCFLAGS="-qmkl=cluster" --enable-openmp && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ - ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ + cp -r /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ From f54360ecb59686477a6d53318f10454ff112e9bc Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sun, 19 Nov 2023 21:49:27 +0800 Subject: [PATCH 09/21] unuse elpa in CI, update elpa intel image daily --- Dockerfile.intel | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 3fba439dd64..452d7220491 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -12,24 +12,29 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO # To save disk space, only install the essential components, but not the whole toolkit. RUN apt-get update && \ - sudo apt -y install intel-basekit intel-hpckit + apt-get install -y \ + intel-oneapi-compiler-dpcpp-cpp \ + intel-oneapi-compiler-fortran \ + intel-oneapi-mkl-devel \ + intel-oneapi-mpi-devel \ + intel-oneapi-vtune RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html -RUN source /opt/intel/oneapi/setvars.sh && \ - cd /tmp && \ - ELPA_VER=2021.05.002 && \ - wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ - tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ - cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure FCFLAGS="-qmkl=cluster" --enable-openmp && \ - make -j`nproc` && \ - make PREFIX=/usr/local install && \ - cp -r /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ - cd /tmp && rm -rf elpa-$ELPA_VER +#RUN source /opt/intel/oneapi/setvars.sh && \ +# cd /tmp && \ +# ELPA_VER=2022.11.001 && \ +# wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ +# tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ +# cd elpa-$ELPA_VER && mkdir build && cd build && \ +# ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ +# make -j`nproc` && \ +# make PREFIX=/usr/local install && \ +# ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ +# cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ --no-check-certificate --quiet -O libtorch.zip && \ @@ -42,7 +47,7 @@ ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/deve RUN source /opt/intel/oneapi/setvars.sh && \ git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ cd abacus-develop && \ - cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON && \ + cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DUSE_ELPA=OFF && \ cmake --build build -j`nproc` && \ cmake --install build && \ rm -rf build && \ From 28173ea313bcd7cec1a844a0f9d295b004224007 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sun, 19 Nov 2023 23:21:19 +0800 Subject: [PATCH 10/21] try disable elpa test --- Dockerfile.intel | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 452d7220491..20f6e3ed8d3 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -24,17 +24,17 @@ SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html -#RUN source /opt/intel/oneapi/setvars.sh && \ -# cd /tmp && \ -# ELPA_VER=2022.11.001 && \ -# wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ -# tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ -# cd elpa-$ELPA_VER && mkdir build && cd build && \ -# ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ -# make -j`nproc` && \ -# make PREFIX=/usr/local install && \ -# ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ -# cd /tmp && rm -rf elpa-$ELPA_VER +RUN source /opt/intel/oneapi/setvars.sh && \ + cd /tmp && \ + ELPA_VER=2022.11.001 && \ + wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ + tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ + cd elpa-$ELPA_VER && mkdir build && cd build && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --disable-runtime-threading-support-checks && \ + make -j`nproc` && \ + make PREFIX=/usr/local install && \ + ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ + cd /tmp && rm -rf elpa-$ELPA_VER RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ --no-check-certificate --quiet -O libtorch.zip && \ From 743197c05880867c55d4946fde2f583a669dd151 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Sun, 19 Nov 2023 23:23:38 +0800 Subject: [PATCH 11/21] try enable avx512 for elpa --- Dockerfile.intel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 20f6e3ed8d3..3e2e9548151 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -30,7 +30,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --disable-runtime-threading-support-checks && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --enable-avx512 && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ @@ -47,7 +47,7 @@ ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/deve RUN source /opt/intel/oneapi/setvars.sh && \ git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ cd abacus-develop && \ - cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DUSE_ELPA=OFF && \ + cmake -B build -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON && \ cmake --build build -j`nproc` && \ cmake --install build && \ rm -rf build && \ From 0517fcca35fde36394836134850372886bd1ffbb Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 00:01:13 +0800 Subject: [PATCH 12/21] try full configure command --- Dockerfile.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 3e2e9548151..360ae3e8525 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -30,7 +30,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp --enable-avx512 && \ + ../configure CFLAGS="-O3 -march=skylake-avx512 -mfma -funsafe-loop-optimizations -funsafe-math-optimizations -ftree-vect-loop-version -ftree-vectorize" FCFLAGS="-O3 -qmkl=cluster -march=skylake-avx512 -mfma" --enable-openmp --enable-option-checking=fatal --enable-avx2 --enable-avx512 && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ From ef69b02797cb81d1d1a249898d0f481a57f8cbdb Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 00:04:52 +0800 Subject: [PATCH 13/21] try full configure command --- Dockerfile.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 360ae3e8525..9ae3bac7ece 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -30,7 +30,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=skylake-avx512 -mfma -funsafe-loop-optimizations -funsafe-math-optimizations -ftree-vect-loop-version -ftree-vectorize" FCFLAGS="-O3 -qmkl=cluster -march=skylake-avx512 -mfma" --enable-openmp --enable-option-checking=fatal --enable-avx2 --enable-avx512 && \ + ../configure FCFLAGS="-O3 -xCORE-AVX512" CFLAGS="-O3 -xCORE-AVX512" --enable-option-checking=fatal SCALAPACK_LDFLAGS="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_lp64 -lpthread " SCALAPACK_FCFLAGS="-I$MKL_HOME/include/intel64/lp64" --enable-avx2 --enable-avx512 && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ From a9a6375707925da5b31d468c305812d4375d1cf7 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 08:27:00 +0800 Subject: [PATCH 14/21] try another flags to disable avx512 --- Dockerfile.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 9ae3bac7ece..ba3d2fb047d 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -30,7 +30,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure FCFLAGS="-O3 -xCORE-AVX512" CFLAGS="-O3 -xCORE-AVX512" --enable-option-checking=fatal SCALAPACK_LDFLAGS="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lmkl_blacs_intelmpi_lp64 -lpthread " SCALAPACK_FCFLAGS="-I$MKL_HOME/include/intel64/lp64" --enable-avx2 --enable-avx512 && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-option-checking=fatal --enable-avx2=fatal --enable-avx512=fatal --enable-openmp && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ From caef35209070966d066d374aad7337fee16ab57f Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 11:12:33 +0800 Subject: [PATCH 15/21] change machine of image and container tests --- .github/workflows/devcontainer.yml | 2 +- .github/workflows/image.yml | 2 +- Dockerfile.intel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 6496a3251aa..f8442b6c76f 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -8,7 +8,7 @@ on: jobs: build_container_and_push: - runs-on: ubuntu-latest + runs-on: self-hosted if: github.repository_owner == 'deepmodeling' strategy: matrix: diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 9203a72f6c8..d4836ddd061 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -7,7 +7,7 @@ on: - 'v*' jobs: build_container_and_push: - runs-on: ubuntu-latest + runs-on: self-hosted if: github.repository_owner == 'deepmodeling' steps: - name: Checkout diff --git a/Dockerfile.intel b/Dockerfile.intel index ba3d2fb047d..f4e2a042ff2 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -30,7 +30,7 @@ RUN source /opt/intel/oneapi/setvars.sh && \ wget -q https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/$ELPA_VER/elpa-$ELPA_VER.tar.gz && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ - ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-option-checking=fatal --enable-avx2=fatal --enable-avx512=fatal --enable-openmp && \ + ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ make -j`nproc` && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ From 1394e7c02866911f1fc69065538e002ce30b4dad Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 13:46:59 +0800 Subject: [PATCH 16/21] Fix: build error with intel image --- .github/workflows/build_test_cmake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index f9957c6b036..b16848bce08 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -46,6 +46,7 @@ jobs: - name: Build run: | + source ~/.bashrc git config --global --add safe.directory `pwd` cmake -B build ${{ matrix.build_args }} cmake --build build -j2 From 53b379ebb63b4a81d2e5cdb19e2a0e8fd8256e4c Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 14:19:35 +0800 Subject: [PATCH 17/21] Fix: build error with intel image --- .github/workflows/build_test_cmake.yml | 2 +- .github/workflows/build_test_makefile.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index b16848bce08..252fca7f668 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -46,7 +46,7 @@ jobs: - name: Build run: | - source ~/.bashrc + source /root/.bashrc git config --global --add safe.directory `pwd` cmake -B build ${{ matrix.build_args }} cmake --build build -j2 diff --git a/.github/workflows/build_test_makefile.yml b/.github/workflows/build_test_makefile.yml index e2129a707f0..53483fa372a 100644 --- a/.github/workflows/build_test_makefile.yml +++ b/.github/workflows/build_test_makefile.yml @@ -19,6 +19,7 @@ jobs: uses: actions/checkout@v4 - name: Build run: | + source /opt/intel/oneapi/setvars.sh export I_MPI_CXX=icpx cd source make -j2 ${{ matrix.build_args }} From 4611670460cd2b69708bf05241cd8d543fc3ba11 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Mon, 20 Nov 2023 15:01:53 +0800 Subject: [PATCH 18/21] reset the paths in Dockerfile.intel --- .github/workflows/build_test_cmake.yml | 1 - .github/workflows/build_test_makefile.yml | 1 - Dockerfile.intel | 11 ++++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index 252fca7f668..f9957c6b036 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -46,7 +46,6 @@ jobs: - name: Build run: | - source /root/.bashrc git config --global --add safe.directory `pwd` cmake -B build ${{ matrix.build_args }} cmake --build build -j2 diff --git a/.github/workflows/build_test_makefile.yml b/.github/workflows/build_test_makefile.yml index 53483fa372a..e2129a707f0 100644 --- a/.github/workflows/build_test_makefile.yml +++ b/.github/workflows/build_test_makefile.yml @@ -19,7 +19,6 @@ jobs: uses: actions/checkout@v4 - name: Build run: | - source /opt/intel/oneapi/setvars.sh export I_MPI_CXX=icpx cd source make -j2 ${{ matrix.build_args }} diff --git a/Dockerfile.intel b/Dockerfile.intel index f4e2a042ff2..6cac8c9f5f6 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -19,7 +19,16 @@ RUN apt-get update && \ intel-oneapi-mpi-devel \ intel-oneapi-vtune -RUN echo "source /opt/intel/oneapi/setvars.sh" >> ${HOME}/.bashrc + +ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/latest' \ + LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib/:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \ + LD_LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib:/opt/intel/oneapi/itac/latest/slib:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/debugger/latest/opt/debugger/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/opt/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/latest/opt/compiler/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \ + PATH=/opt/intel/oneapi/vtune/latest/bin64:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mpi/latest/bin:/opt/intel/oneapi/mkl/latest/bin/:/opt/intel/oneapi/itac/latest/bin:/opt/intel/oneapi/inspector/latest/bin64:/opt/intel/oneapi/dpcpp-ct/latest/bin:/opt/intel/oneapi/dev-utilities/latest/bin:/opt/intel/oneapi/debugger/latest/opt/debugger/bin:/opt/intel/oneapi/compiler/latest/opt/oclfpga/bin:/opt/intel/oneapi/compiler/latest/bin:/opt/intel/oneapi/advisor/latest/bin64:/opt/mamba/bin:/opt/mamba/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + MKLROOT='/opt/intel/oneapi/mkl/latest' \ + FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib/prov:/usr/lib/x86_64-linux-gnu/libfabric' \ + CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/latest/env/..:/opt/intel/oneapi/mkl/latest/lib/cmake:/opt/intel/oneapi/ipp/latest/lib/cmake/ipp:/opt/intel/oneapi/dpl/latest/lib/cmake/oneDPL:/opt/intel/oneapi/dnnl/latest/lib/cmake:/opt/intel/oneapi/dal/latest:/opt/intel/oneapi/compiler/latest' \ + CMPLR_ROOT='/opt/intel/oneapi/compiler/latest' + SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx From f71347d203a45b63b09dbc7db819dbaf9cbcc766 Mon Sep 17 00:00:00 2001 From: Chun Cai Date: Mon, 20 Nov 2023 21:37:30 +0800 Subject: [PATCH 19/21] remove test commands --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index f8442b6c76f..597283930b8 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -4,7 +4,6 @@ on: push: branches: - develop - - container_test jobs: build_container_and_push: From 3468a251e640fa57be15ec4ac105aacaf6dedfc4 Mon Sep 17 00:00:00 2001 From: dyzheng Date: Tue, 21 Nov 2023 12:59:02 +0800 Subject: [PATCH 20/21] Fix: run build_test_cmake on self-hosted --- .github/workflows/build_test_cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index f9957c6b036..67ad0111b28 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -5,7 +5,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: include: From b66424f939bca9394997aaf3b1c892ea4a0ccbbe Mon Sep 17 00:00:00 2001 From: dyzheng Date: Tue, 21 Nov 2023 15:04:26 +0800 Subject: [PATCH 21/21] only test PAW with GNU toolchain --- .github/workflows/build_test_cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index 67ad0111b28..8065605e323 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -5,7 +5,7 @@ on: jobs: test: - runs-on: self-hosted + runs-on: ubuntu-latest strategy: matrix: include: @@ -20,7 +20,7 @@ jobs: build_args: "-DENABLE_LIBXC=1 -DENABLE_DEEPKS=1 -DENABLE_LIBRI=1 -DENABLE_PAW=1" name: "Build extra components with GNU toolchain" - tag: intel - build_args: "-DENABLE_LIBXC=1 -DENABLE_DEEPKS=1 -DENABLE_LIBRI=1 -DENABLE_PAW=1" + build_args: "-DENABLE_LIBXC=1 -DENABLE_DEEPKS=1 -DENABLE_LIBRI=1" name: "Build extra components with Intel toolchain" - tag: cuda