From 966c913f3aff5c2ada0378cace1f03a88bb01b92 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 14:49:00 +0800
Subject: [PATCH 01/17] Update install_elpa.sh

update elpa version to 2023.05.001
---
 toolchain/scripts/stage3/install_elpa.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/toolchain/scripts/stage3/install_elpa.sh b/toolchain/scripts/stage3/install_elpa.sh
index 59c2a72fa58..7b112c7070b 100755
--- a/toolchain/scripts/stage3/install_elpa.sh
+++ b/toolchain/scripts/stage3/install_elpa.sh
@@ -7,10 +7,12 @@
 SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_NAME")/.." && pwd -P)"
 
 # From https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html
-elpa_ver="2021.11.002"
-elpa_sha256="576f1caeed7883b81396640fda0f504183866cf6cbd4bc71d1383ba2208f1f97" # 2021.11.002, by sha256sum
+elpa_ver="2023.05.001"
+elpa_sha256="ec64be5d6522810d601a3b8e6a31720e3c3eb4af33a434d8a64570d76e6462b6"
+#elpa_ver="2021.11.002"
+#elpa_sha256="576f1caeed7883b81396640fda0f504183866cf6cbd4bc71d1383ba2208f1f97"
 #elpa_ver="2022.11.001"
-# elpa_sha256="35e397d7c0af95bb43bc7bef7fff29425c1da400fa0cd86ae8d3bd2ff2f9d999" # 2022.11.001
+#elpa_sha256="35e397d7c0af95bb43bc7bef7fff29425c1da400fa0cd86ae8d3bd2ff2f9d999"
 
 
 source "${SCRIPT_DIR}"/common_vars.sh

From f52d3556fcfd055f6c239b2a67d0ff2b978db074 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 14:57:31 +0800
Subject: [PATCH 02/17] Update cmake version to 3.27.6

---
 toolchain/scripts/stage0/install_cmake.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/toolchain/scripts/stage0/install_cmake.sh b/toolchain/scripts/stage0/install_cmake.sh
index ed6dcba35a4..6f08a62b1df 100755
--- a/toolchain/scripts/stage0/install_cmake.sh
+++ b/toolchain/scripts/stage0/install_cmake.sh
@@ -20,13 +20,13 @@ cd "${BUILDDIR}"
 case "${with_cmake}" in
   __INSTALL__)
     echo "==================== Installing CMake ===================="
-    cmake_ver="3.26.3"
+    cmake_ver="3.27.6"
     if [ "${OPENBLAS_ARCH}" = "arm64" ]; then
       cmake_arch="linux-aarch64"
-      cmake_sha256="b002c22b926aacd6fefe64bcf08620216088eb72f55ac532b7bcfd4d93443d50"
+      cmake_sha256="a83e01ed1cdf44c2e33e0726513b9a35a8c09e3b5a126fd720b3c8a9d5552368"
     elif [ "${OPENBLAS_ARCH}" = "x86_64" ]; then
       cmake_arch="linux-x86_64"
-      cmake_sha256="8ec0ef24375a1d0e78de2f790b4545d0718acc55fd7e2322ecb8e135696c77fe"
+      cmake_sha256="8c449dabb2b2563ec4e6d5e0fb0ae09e729680efab71527b59015131cea4a042"
     else
       report_error ${LINENO} \
         "cmake installation for ARCH=${ARCH} is not supported. You can try to use the system installation using the flag --with-cmake=system instead."

From c3ad5ec93f95752524545653fdeb99b6771e71f1 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 14:57:42 +0800
Subject: [PATCH 03/17] Update gcc version to 13.2.0

---
 toolchain/scripts/stage0/install_gcc.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toolchain/scripts/stage0/install_gcc.sh b/toolchain/scripts/stage0/install_gcc.sh
index 13b8d8c7804..2effe52ca79 100755
--- a/toolchain/scripts/stage0/install_gcc.sh
+++ b/toolchain/scripts/stage0/install_gcc.sh
@@ -6,8 +6,8 @@
 [ "${BASH_SOURCE[0]}" ] && SCRIPT_NAME="${BASH_SOURCE[0]}" || SCRIPT_NAME=$0
 SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_NAME")/.." && pwd -P)"
 
-gcc_ver="13.1.0"
-gcc_sha256="bacd4c614d8bd5983404585e53478d467a254249e0f1bb747c8bc6d787bd4fa2"
+gcc_ver="13.2.0"
+gcc_sha256="8cb4be3796651976f94b9356fa08d833524f62420d6292c5033a9a26af315078"
 
 source "${SCRIPT_DIR}"/common_vars.sh
 source "${SCRIPT_DIR}"/tool_kit.sh

From 8f1cc77661599559e8a0c32ed126fd97e65fd345 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 14:58:26 +0800
Subject: [PATCH 04/17] Update install_openblas.sh

- update OpenBLAS version to 0.3.24
- update NUM_THREADS to 128 in compiling
---
 toolchain/scripts/stage2/install_openblas.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/toolchain/scripts/stage2/install_openblas.sh b/toolchain/scripts/stage2/install_openblas.sh
index fd11d4dc2a0..c4b922d51be 100755
--- a/toolchain/scripts/stage2/install_openblas.sh
+++ b/toolchain/scripts/stage2/install_openblas.sh
@@ -6,8 +6,8 @@
 [ "${BASH_SOURCE[0]}" ] && SCRIPT_NAME="${BASH_SOURCE[0]}" || SCRIPT_NAME=$0
 SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_NAME")/.." && pwd -P)"
 
-openblas_ver="0.3.23" # Keep in sync with get_openblas_arch.sh
-openblas_sha256="5d9491d07168a5d00116cdc068a40022c3455bf9293c7cb86a65b1054d7e5114"
+openblas_ver="0.3.24" # Keep in sync with get_openblas_arch.sh
+openblas_sha256="ceadc5065da97bd92404cac7254da66cc6eb192679cf1002098688978d4d5132"
 openblas_pkg="OpenBLAS-${openblas_ver}.tar.gz"
 
 source "${SCRIPT_DIR}"/common_vars.sh
@@ -76,7 +76,7 @@ case "${with_openblas}" in
         make -j $(get_nprocs) \
           MAKE_NB_JOBS=0 \
           TARGET=${TARGET} \
-          NUM_THREADS=64 \
+          NUM_THREADS=128 \
           USE_THREAD=1 \
           USE_OPENMP=1 \
           NO_AFFINITY=1 \
@@ -88,7 +88,7 @@ case "${with_openblas}" in
         make -j $(get_nprocs) \
           MAKE_NB_JOBS=0 \
           TARGET=NEHALEM \
-          NUM_THREADS=64 \
+          NUM_THREADS=128 \
           USE_THREAD=1 \
           USE_OPENMP=1 \
           NO_AFFINITY=1 \
@@ -100,7 +100,7 @@ case "${with_openblas}" in
       make -j $(get_nprocs) \
         MAKE_NB_JOBS=0 \
         TARGET=${TARGET} \
-        NUM_THREADS=64 \
+        NUM_THREADS=128 \
         USE_THREAD=1 \
         USE_OPENMP=1 \
         NO_AFFINITY=1 \

From 1acdfd402e6fb516bdbb5c32eb84fae27ae80ed7 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:22:22 +0800
Subject: [PATCH 05/17] Set default intel compiler to `icpx`

---
 toolchain/build_abacus_intel.sh       | 12 ++++++------
 toolchain/install_abacus_toolchain.sh |  6 +++---
 toolchain/toolchain_intel-mpich.sh    |  6 +++---
 toolchain/toolchain_intel.sh          |  9 +++++----
 4 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/toolchain/build_abacus_intel.sh b/toolchain/build_abacus_intel.sh
index c745c90fd5f..84f8a791ec1 100755
--- a/toolchain/build_abacus_intel.sh
+++ b/toolchain/build_abacus_intel.sh
@@ -26,15 +26,15 @@ PREFIX=$ABACUS_DIR
 ELPA=$INSTALL_DIR/elpa-2021.11.002/cpu
 CEREAL=$INSTALL_DIR/cereal-1.3.2/include/cereal
 LIBXC=$INSTALL_DIR/libxc-6.2.2
-LIBTORCH=$INSTALL_DIR/libtorch-2.0.1/share/cmake/Torch
-LIBNPY=$INSTALL_DIR/libnpy-0.1.0/include
+# LIBTORCH=$INSTALL_DIR/libtorch-2.0.1/share/cmake/Torch
+# LIBNPY=$INSTALL_DIR/libnpy-0.1.0/include
 # LIBRI=$INSTALL_DIR/LibRI-0.1.0
 # LIBCOMM=$INSTALL_DIR/LibComm-0.1.0
 # DEEPMD=$HOME/apps/anaconda3/envs/deepmd
 
 # if use deepks and deepmd
 cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
-        -DCMAKE_CXX_COMPILER=icpc \
+        -DCMAKE_CXX_COMPILER=icpx \
         -DMPI_CXX_COMPILER=mpiicpc \
         -DMKLROOT=$MKLROOT \
         -DELPA_DIR=$ELPA \
@@ -46,9 +46,9 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
         -DUSE_OPENMP=ON \
         -DENABLE_ASAN=OFF \
         -DUSE_ELPA=ON \
-        -DENABLE_DEEPKS=1 \
-        -DTorch_DIR=$LIBTORCH \
-        -Dlibnpy_INCLUDE_DIR=$LIBNPY \
+#         -DENABLE_DEEPKS=1 \
+#         -DTorch_DIR=$LIBTORCH \
+#         -Dlibnpy_INCLUDE_DIR=$LIBNPY \
 #         -DENABLE_LIBRI=ON \
 #         -DLIBRI_DIR=$LIBRI \
 #         -DLIBCOMM_DIR=$LIBCOMM \
diff --git a/toolchain/install_abacus_toolchain.sh b/toolchain/install_abacus_toolchain.sh
index 0f5c1b65722..be416873074 100755
--- a/toolchain/install_abacus_toolchain.sh
+++ b/toolchain/install_abacus_toolchain.sh
@@ -118,7 +118,7 @@ The --enable-FEATURE options follow the rules:
   --enable-FEATURE=no     Disable this particular feature
   --enable-FEATURE        The option keyword alone is equivalent to
                           --enable-FEATURE=yes
-
+  ===== NOTICE: THESE FEATURE AER NOT INCLUDED IN ABACUS =====
   --enable-cuda           Turn on GPU (CUDA) support (can be combined
                           with --enable-opencl).
                           Default = no
@@ -302,9 +302,9 @@ enable_tsan="__FALSE__"
 enable_opencl="__FALSE__"
 enable_cuda="__FALSE__"
 enable_hip="__FALSE__"
-export intel_classic="yes" 
+export intel_classic="no" 
 # no, then icc->icx, icpc->icpx, 
-# which cannot compile elpa-2021 and fftw.3.3.10 in some place
+# which cannot compile elpa in AMD server
 # due to some so-called cross-compile problem
 # and will lead to problem in force calculation
 # but icx is recommended by intel compiler
diff --git a/toolchain/toolchain_intel-mpich.sh b/toolchain/toolchain_intel-mpich.sh
index b40d0f3aec2..f6f14cecc44 100755
--- a/toolchain/toolchain_intel-mpich.sh
+++ b/toolchain/toolchain_intel-mpich.sh
@@ -21,9 +21,9 @@
 --with-fftw=no \
 --with-elpa=install \
 --with-cereal=install \
---with-libtorch=install \
---with-libnpy=install \
+--with-libtorch=no \
+--with-libnpy=no \
 --with-libri=no \
 --with-libcomm=no \
---with-intel-classic=yes \
+--with-intel-classic=no \
 | tee compile.log
\ No newline at end of file
diff --git a/toolchain/toolchain_intel.sh b/toolchain/toolchain_intel.sh
index 1253bfd1b46..c433aee576c 100755
--- a/toolchain/toolchain_intel.sh
+++ b/toolchain/toolchain_intel.sh
@@ -22,9 +22,10 @@
 --with-fftw=no \
 --with-elpa=install \
 --with-cereal=install \
---with-libtorch=install \
---with-libnpy=install \
+--with-libtorch=no \
+--with-libnpy=no \
 --with-libri=no \
 --with-libcomm=no \
---with-intel-classic=yes \
-| tee compile.log
\ No newline at end of file
+--with-intel-classic=no \
+| tee compile.log
+# if you are using AMD server: set --with-intel-classic=yes
\ No newline at end of file

From 19523c87c2f99eb8c96e1dc5cc7ac88ccaf95c47 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:36:53 +0800
Subject: [PATCH 06/17] update build.sh scripts

---
 toolchain/build_abacus_gnu.sh         | 10 ++++++----
 toolchain/build_abacus_intel-mpich.sh | 16 +++++++++-------
 toolchain/build_abacus_intel.sh       |  4 +++-
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/toolchain/build_abacus_gnu.sh b/toolchain/build_abacus_gnu.sh
index 259facdf32e..a8e0e27c4b0 100755
--- a/toolchain/build_abacus_gnu.sh
+++ b/toolchain/build_abacus_gnu.sh
@@ -22,9 +22,9 @@ BUILD_DIR=build_abacus
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR
-LAPACK=$INSTALL_DIR/openblas-0.3.23/lib
+LAPACK=$INSTALL_DIR/openblas-0.3.24/lib
 SCALAPACK=$INSTALL_DIR/scalapalack-2.2.1/lib
-ELPA=$INSTALL_DIR/elpa-2021.11.002/cpu
+ELPA=$INSTALL_DIR/elpa-2023.05.001/cpu
 FFTW3=$INSTALL_DIR/fftw-3.3.10
 CEREAL=$INSTALL_DIR/cereal-1.3.2/include/cereal
 LIBXC=$INSTALL_DIR/libxc-6.2.2
@@ -58,10 +58,12 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
 # 	      -DTensorFlow_DIR=$DEEPMD \
 
 # # add mkl env for libtorch to link
-# # gnu-toolchain will lack of -lmkl when load libtorch
-# # need to fix -- zhaoqing in 2023-09-02
+# if one want to install libtorch, mkl should be load in build process
+# for -lmkl when load libtorch
 # module load mkl
 
+# if one want's to include deepmd, your gcc version should be >= 11.3.0
+
 cmake --build $BUILD_DIR -j `nproc` 
 cmake --install $BUILD_DIR 
 
diff --git a/toolchain/build_abacus_intel-mpich.sh b/toolchain/build_abacus_intel-mpich.sh
index a2daff3bd56..6e231f3fa27 100755
--- a/toolchain/build_abacus_intel-mpich.sh
+++ b/toolchain/build_abacus_intel-mpich.sh
@@ -4,7 +4,7 @@
 #SBATCH -n 16
 #SBATCH -o install.log
 #SBATCH -e install.err
-# install ABACUS with libxc and deepks
+# build and install ABACUS with libxc, also can with deepks and deepmd
 # JamesMisaka in 2023.08.31
 
 # Build ABACUS by intel-toolchain with mpich
@@ -23,11 +23,11 @@ BUILD_DIR=build_abacus
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR
-ELPA=$INSTALL_DIR/elpa-2021.11.002/cpu
+ELPA=$INSTALL_DIR/elpa-2023.05.001/cpu
 CEREAL=$INSTALL_DIR/cereal-1.3.2/include/cereal
 LIBXC=$INSTALL_DIR/libxc-6.2.2
-LIBTORCH=$INSTALL_DIR/libtorch-2.0.1/share/cmake/Torch
-LIBNPY=$INSTALL_DIR/libnpy-0.1.0/include
+# LIBTORCH=$INSTALL_DIR/libtorch-2.0.1/share/cmake/Torch
+# LIBNPY=$INSTALL_DIR/libnpy-0.1.0/include
 # LIBRI=$INSTALL_DIR/LibRI-0.1.0
 # LIBCOMM=$INSTALL_DIR/LibComm-0.1.0
 # DEEPMD=$HOME/apps/anaconda3/envs/deepmd
@@ -44,15 +44,17 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
         -DUSE_OPENMP=ON \
         -DENABLE_ASAN=OFF \
         -DUSE_ELPA=ON \
-        -DENABLE_DEEPKS=1 \
-        -DTorch_DIR=$LIBTORCH \
-        -Dlibnpy_INCLUDE_DIR=$LIBNPY \
+        # -DENABLE_DEEPKS=1 \
+        # -DTorch_DIR=$LIBTORCH \
+        # -Dlibnpy_INCLUDE_DIR=$LIBNPY \
         # -DENABLE_LIBRI=ON \
         # -DLIBRI_DIR=$LIBRI \
         # -DLIBCOMM_DIR=$LIBCOMM \
 	    # -DDeePMD_DIR=$DEEPMD \
 	    # -DTensorFlow_DIR=$DEEPMD \
 
+# if one want's to include deepmd, your gcc version should be >= 11.3.0
+
 cmake --build $BUILD_DIR -j `nproc` 
 cmake --install $BUILD_DIR 
 
diff --git a/toolchain/build_abacus_intel.sh b/toolchain/build_abacus_intel.sh
index 84f8a791ec1..44e0539e5c2 100755
--- a/toolchain/build_abacus_intel.sh
+++ b/toolchain/build_abacus_intel.sh
@@ -23,7 +23,7 @@ BUILD_DIR=build_abacus
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR
-ELPA=$INSTALL_DIR/elpa-2021.11.002/cpu
+ELPA=$INSTALL_DIR/elpa-2023.05.001/cpu
 CEREAL=$INSTALL_DIR/cereal-1.3.2/include/cereal
 LIBXC=$INSTALL_DIR/libxc-6.2.2
 # LIBTORCH=$INSTALL_DIR/libtorch-2.0.1/share/cmake/Torch
@@ -58,6 +58,8 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
 cmake --build $BUILD_DIR -j `nproc` 
 cmake --install $BUILD_DIR 
 
+# if one want's to include deepmd, your gcc version should be >= 11.3.0
+
 # generate abacus_env.sh
 cat << EOF > "${TOOL}/abacus_env.sh"
 source $INSTALL_DIR/setup

From c61e8b7c6a94e2499e7eb4003c62808c36374fcc Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:37:01 +0800
Subject: [PATCH 07/17] Update README.md

---
 toolchain/README.md | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index b7702a55b78..755d06ce782 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -1,5 +1,5 @@
 # The ABACUS Toolchain
-Version 2023.4
+Version 2023.5
 
 ## Author
 [QuantumMisaka](https://github.com/QuantumMisaka) 
@@ -31,13 +31,14 @@ and give setup files that you can use to compile ABACUS.
 - [ ] Better compliation method for ABACUS-DEEPMD and ABACUS-DEEPKS.
 - [ ] A better `setup` and toolchain code structure.
 - [ ] Modulefile generation scripts.
-- [ ] Support for `acml` toolchain (scripts are partly in toolchain now) or other AMD compiler and math lib like `AOCL` and `AOCC`
+- [ ] Support for AMD compiler and math lib like `AOCL` and `AOCC`
 
 
 ## Usage Online & Offline
 Main script is `install_abacus_toolchain.sh`, 
 which will use scripts in `scripts` directory 
 to compile install dependencies of ABACUS.
+You can just `./install_abacus_toolchain.sh -h` to get more help message.
 
 **Notice: You SHOULD `source` or `module load` related environments before use toolchain method for installation, espacially for `gcc` or `intel-oneAPI` !!!! for example, `module load mkl mpi icc compiler`**
 
@@ -146,6 +147,7 @@ Users can get help messages by simply:
 
 
 ## Common Problem and Solution
+### shell problem
 If you encounter problem like:
 ```shell
 /bin/bash^M: bad interpreter: No such file or directory
@@ -157,6 +159,22 @@ or   `permission denied` problem, you can simply run:
 And also, you can fix `permission denied` problem via `chmod +x`
 if `pre_set.sh` have no execution permission.
 
+### libtorch and deepks problem
+If deepks feature have problem, you can manually change libtorch version
+from 2.0.1 to 1.12.0 in `toolchain/scripts/stage4/install_libtorch.sh`.
+Also, you can install ABACUS without deepks by removing all the deepks and related options.
+
+
+### LibRI and LibComm problem
+(There is some problem sometimes when compling with LibRI and LibComm, detailed information is needed)
+
+
+### Intel-oneAPI problem
+Sometimes Intel-oneAPI have problem to link `mpirun`, 
+which will always show in 2023.2.0 version of MPI in Intel-oneAPI. 
+Try `source /path/to/setvars.sh` or install another version of IntelMPI may help.
+
+
 
 ## Advanced Installation Usage
 

From 94e16ee4d35c1cfdfdf523b032c1843028f903b4 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:37:25 +0800
Subject: [PATCH 08/17] Update README.md

---
 toolchain/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index 755d06ce782..2b793bcb591 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -79,7 +79,7 @@ just by using this toolchain
 > cp ***.tar.gz build/
 ```
 
-Notice: for `CEREAL`, `LibNPY`, `LibRI` and `LibCOMM`, 
+Notice: for `CEREAL`, `Libnpy`, `LibRI` and `LibComm`, 
 you need to download them from github.com, 
 rename it as formatted, and put them in `build` directory at the same time
 e.g.:

From d66165e33496645d2fa4587ad49da3d922e331b9 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:39:14 +0800
Subject: [PATCH 09/17] modified toolchain*.sh

---
 toolchain/toolchain_gnu.sh         | 2 +-
 toolchain/toolchain_intel-mpich.sh | 4 ++--
 toolchain/toolchain_intel.sh       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/toolchain/toolchain_gnu.sh b/toolchain/toolchain_gnu.sh
index c12066ae396..57105f601b7 100755
--- a/toolchain/toolchain_gnu.sh
+++ b/toolchain/toolchain_gnu.sh
@@ -6,7 +6,7 @@
 #SBATCH -e compile.err
 
 # JamesMisaka in 2023-09-16
-# install abacus by gnu-toolchain
+# install abacus dependency by gnu-toolchain
 # one can use mpich or openmpi
 # libtorch and libnpy are for deepks support, which can be =no
 
diff --git a/toolchain/toolchain_intel-mpich.sh b/toolchain/toolchain_intel-mpich.sh
index f6f14cecc44..2368e3964f8 100755
--- a/toolchain/toolchain_intel-mpich.sh
+++ b/toolchain/toolchain_intel-mpich.sh
@@ -6,8 +6,8 @@
 #SBATCH -e compile.err
 
 # JamesMisaka in 2023-09-16
-# install abacus by intel-toolchain 
-# use mkl , and mpich instead of intelmpi
+# install abacus dependency by intel-toolchain 
+# use mkl ,and mpich instead of intelmpi
 # libtorch and libnpy are for deepks support, which can be =no
 
 # module load mkl compiler
diff --git a/toolchain/toolchain_intel.sh b/toolchain/toolchain_intel.sh
index c433aee576c..8f391be008d 100755
--- a/toolchain/toolchain_intel.sh
+++ b/toolchain/toolchain_intel.sh
@@ -6,7 +6,7 @@
 #SBATCH -e compile.err
 
 # JamesMisaka in 2023-08-31
-# install abacus by intel-toolchain
+# install abacus dependency by intel-toolchain
 # use mkl and intelmpi
 # but mpich and openmpi can also be tried
 # libtorch and libnpy are for deepks support, which can be =no

From 8bfccec67717024d76d0d940aeaf0b533378eb1a Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:39:52 +0800
Subject: [PATCH 10/17] Update toolchain_intel-mpich.sh

---
 toolchain/toolchain_intel-mpich.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/toolchain/toolchain_intel-mpich.sh b/toolchain/toolchain_intel-mpich.sh
index 2368e3964f8..ffc26266703 100755
--- a/toolchain/toolchain_intel-mpich.sh
+++ b/toolchain/toolchain_intel-mpich.sh
@@ -26,4 +26,5 @@
 --with-libri=no \
 --with-libcomm=no \
 --with-intel-classic=no \
-| tee compile.log
\ No newline at end of file
+| tee compile.log
+# if you are using AMD server: set --with-intel-classic=yes
\ No newline at end of file

From bb37196b55f4b1bed625e211ea7beb64fc7580c2 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:48:03 +0800
Subject: [PATCH 11/17] remove error message in build*.sh when do install

---
 toolchain/build_abacus_gnu.sh         | 3 +--
 toolchain/build_abacus_intel-mpich.sh | 5 ++---
 toolchain/build_abacus_intel.sh       | 4 +---
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/toolchain/build_abacus_gnu.sh b/toolchain/build_abacus_gnu.sh
index a8e0e27c4b0..6d65325331a 100755
--- a/toolchain/build_abacus_gnu.sh
+++ b/toolchain/build_abacus_gnu.sh
@@ -46,7 +46,6 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
         -DENABLE_LCAO=ON \
         -DENABLE_LIBXC=ON \
         -DUSE_OPENMP=ON \
-        -DENABLE_ASAN=OFF \
         -DUSE_ELPA=ON \
 #         -DENABLE_DEEPKS=1 \
 #         -DTorch_DIR=$LIBTORCH \
@@ -65,7 +64,7 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
 # if one want's to include deepmd, your gcc version should be >= 11.3.0
 
 cmake --build $BUILD_DIR -j `nproc` 
-cmake --install $BUILD_DIR 
+cmake --install $BUILD_DIR 2>/dev/null
 
 # generate abacus_env.sh
 cat << EOF > "${TOOL}/abacus_env.sh"
diff --git a/toolchain/build_abacus_intel-mpich.sh b/toolchain/build_abacus_intel-mpich.sh
index 6e231f3fa27..6f7e156140f 100755
--- a/toolchain/build_abacus_intel-mpich.sh
+++ b/toolchain/build_abacus_intel-mpich.sh
@@ -33,7 +33,7 @@ LIBXC=$INSTALL_DIR/libxc-6.2.2
 # DEEPMD=$HOME/apps/anaconda3/envs/deepmd
 
 cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
-        -DCMAKE_CXX_COMPILER=icpc \
+        -DCMAKE_CXX_COMPILER=icpx \
         -DMPI_CXX_COMPILER=mpicxx \
         -DMKLROOT=$MKLROOT \
         -DELPA_DIR=$ELPA \
@@ -42,7 +42,6 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
         -DENABLE_LCAO=ON \
         -DENABLE_LIBXC=ON \
         -DUSE_OPENMP=ON \
-        -DENABLE_ASAN=OFF \
         -DUSE_ELPA=ON \
         # -DENABLE_DEEPKS=1 \
         # -DTorch_DIR=$LIBTORCH \
@@ -56,7 +55,7 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
 # if one want's to include deepmd, your gcc version should be >= 11.3.0
 
 cmake --build $BUILD_DIR -j `nproc` 
-cmake --install $BUILD_DIR 
+cmake --install $BUILD_DIR 2>/dev/null
 
 # generate abacus_env.sh
 cat << EOF > "${TOOL}/abacus_env.sh"
diff --git a/toolchain/build_abacus_intel.sh b/toolchain/build_abacus_intel.sh
index 44e0539e5c2..7240f986c43 100755
--- a/toolchain/build_abacus_intel.sh
+++ b/toolchain/build_abacus_intel.sh
@@ -42,9 +42,7 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
         -DLibxc_DIR=$LIBXC \
         -DENABLE_LCAO=ON \
         -DENABLE_LIBXC=ON \
-        -DENABLE_LIBRI=OFF \
         -DUSE_OPENMP=ON \
-        -DENABLE_ASAN=OFF \
         -DUSE_ELPA=ON \
 #         -DENABLE_DEEPKS=1 \
 #         -DTorch_DIR=$LIBTORCH \
@@ -56,7 +54,7 @@ cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
 # 	      -DTensorFlow_DIR=$DEEPMD \
 
 cmake --build $BUILD_DIR -j `nproc` 
-cmake --install $BUILD_DIR 
+cmake --install $BUILD_DIR 2>/dev/null
 
 # if one want's to include deepmd, your gcc version should be >= 11.3.0
 

From 307950bbe3b1105bc03b38ff20ce2765efcacbe3 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 15:58:31 +0800
Subject: [PATCH 12/17] Update README.md

---
 toolchain/README.md | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index 2b793bcb591..91b4f2ec524 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -52,7 +52,7 @@ You can just `./install_abacus_toolchain.sh -h` to get more help message.
 
 All packages will be downloaded from [cp2k-static/download](https://www.cp2k.org/static/downloads). by  `wget` , and will be detailedly compiled and installed in `install` directory by toolchain scripts, despite of:
 - `CEREAL` which will be downloaded from [CEREAL](https://github.com/USCiLab/cereal)  
-- `LibNPY` which will be downloaded from [LIBNPY](https://github.com/llohse/libnpy)
+- `Libnpy` which will be downloaded from [LIBNPY](https://github.com/llohse/libnpy)
 - `LibRI` which will be downloaded from [LibRI](https://github.com/abacusmodeling/LibRI)
 - `LibCOMM` which will be downloaded from [LibComm](https://github.com/abacusmodeling/LibComm)
 Notice: These packages will be downloaded by `wget` from `github.com`, which is hard to be done in Chinese Internet. You may need to use offline installation method. 
@@ -79,6 +79,25 @@ just by using this toolchain
 > cp ***.tar.gz build/
 ```
 
+The needed dependencies version default:
+- `cmake` 3.27.6
+- `gcc` 13.2.0 (which will always NOT be installed, But use system)
+- `OpenMPI` 4.1.5
+- `MPICH` 4.1.2
+- `OpenBLAS` 0.3.24 (Intel toolchain need `get_vars.sh` tool from it)
+- `ScaLAPACK` 2.2.1
+- `FFTW` 3.3.10
+- `LibXC` 6.2.2
+- `ELPA` 2023.05.001
+- `CEREAL` 1.3.2
+And Intel-oneAPI need user or server manager to manually install from Intel.
+
+Dependencies below are optional:
+- `LibTorch` 2.0.1
+- `Libnpy` 0.1.0
+- `LibRI` 0.1.0
+- `LibComm` 0.1.0
+
 Notice: for `CEREAL`, `Libnpy`, `LibRI` and `LibComm`, 
 you need to download them from github.com, 
 rename it as formatted, and put them in `build` directory at the same time

From bd476ef8c00626a57f37f751fcb7b56edc71025b Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 16:02:38 +0800
Subject: [PATCH 13/17] minor update

---
 toolchain/README.md                   | 1 +
 toolchain/build_abacus_gnu.sh         | 2 +-
 toolchain/build_abacus_intel-mpich.sh | 2 +-
 toolchain/build_abacus_intel.sh       | 2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index 91b4f2ec524..e5f2cf1730e 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -91,6 +91,7 @@ The needed dependencies version default:
 - `ELPA` 2023.05.001
 - `CEREAL` 1.3.2
 And Intel-oneAPI need user or server manager to manually install from Intel.
+[Intel-oneAPI](https://www.intel.cn/content/www/cn/zh/developer/tools/oneapi/toolkits.html)
 
 Dependencies below are optional:
 - `LibTorch` 2.0.1
diff --git a/toolchain/build_abacus_gnu.sh b/toolchain/build_abacus_gnu.sh
index 6d65325331a..66fb821caf2 100755
--- a/toolchain/build_abacus_gnu.sh
+++ b/toolchain/build_abacus_gnu.sh
@@ -18,7 +18,7 @@ source $INSTALL_DIR/setup
 cd $ABACUS_DIR
 ABACUS_DIR=$(pwd)
 
-BUILD_DIR=build_abacus
+BUILD_DIR=build_abacus_gnu
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR
diff --git a/toolchain/build_abacus_intel-mpich.sh b/toolchain/build_abacus_intel-mpich.sh
index 6f7e156140f..181378804ad 100755
--- a/toolchain/build_abacus_intel-mpich.sh
+++ b/toolchain/build_abacus_intel-mpich.sh
@@ -19,7 +19,7 @@ source $INSTALL_DIR/setup
 cd $ABACUS_DIR
 ABACUS_DIR=$(pwd)
 
-BUILD_DIR=build_abacus
+BUILD_DIR=build_abacus_intel-mpich
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR
diff --git a/toolchain/build_abacus_intel.sh b/toolchain/build_abacus_intel.sh
index 7240f986c43..6be78f20699 100755
--- a/toolchain/build_abacus_intel.sh
+++ b/toolchain/build_abacus_intel.sh
@@ -19,7 +19,7 @@ source $INSTALL_DIR/setup
 cd $ABACUS_DIR
 ABACUS_DIR=$(pwd)
 
-BUILD_DIR=build_abacus
+BUILD_DIR=build_abacus_intel
 rm -rf $BUILD_DIR
 
 PREFIX=$ABACUS_DIR

From 06273dc3672811af951465fc683ff753c5c5be71 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 16:18:18 +0800
Subject: [PATCH 14/17] Update README.md

---
 toolchain/README.md | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index e5f2cf1730e..75a36676e41 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -38,6 +38,7 @@ and give setup files that you can use to compile ABACUS.
 Main script is `install_abacus_toolchain.sh`, 
 which will use scripts in `scripts` directory 
 to compile install dependencies of ABACUS.
+
 You can just `./install_abacus_toolchain.sh -h` to get more help message.
 
 **Notice: You SHOULD `source` or `module load` related environments before use toolchain method for installation, espacially for `gcc` or `intel-oneAPI` !!!! for example, `module load mkl mpi icc compiler`**
@@ -160,12 +161,6 @@ or you can also do it in a more completely way:
 > rm -rf install build/*/* build/OpenBLAS*/ build/setup_*
 ```
 
-Users can get help messages by simply:
-```shell
-> ./install_abacus_toolchain.sh -h # or --help
-```
-
-
 ## Common Problem and Solution
 ### shell problem
 If you encounter problem like:
@@ -184,6 +179,18 @@ If deepks feature have problem, you can manually change libtorch version
 from 2.0.1 to 1.12.0 in `toolchain/scripts/stage4/install_libtorch.sh`.
 Also, you can install ABACUS without deepks by removing all the deepks and related options.
 
+NOTICE: if you want deepks feature, your intel-mkl environment should be accessible in building process. you can check it in `build_abacus_gnu.sh`
+
+### deepmd feature problem
+When you encounter problem like `GLIBCXX_3.4.29 not found`, it is sure that your `gcc` version is lower than the requirement of `libdeepmd`.
+
+After my test, you need `gcc`>11.3.1 to enable deepmd feature in ABACUS.
+
+### ELPA problem via Intel-oneAPI toolchain in AMD server
+The default compiler for Intel-oneAPI is `icpx` and `icx`, which will cause problem when compling ELPA in AMD server.
+
+The best way is to change `icpx` to `icpc`, `icx` to `icc`. user can manually change it in toolchain*.sh via `--with-intel-classic=yes`
+
 
 ### LibRI and LibComm problem
 (There is some problem sometimes when compling with LibRI and LibComm, detailed information is needed)
@@ -194,6 +201,8 @@ Sometimes Intel-oneAPI have problem to link `mpirun`,
 which will always show in 2023.2.0 version of MPI in Intel-oneAPI. 
 Try `source /path/to/setvars.sh` or install another version of IntelMPI may help.
 
+More problem and possible solution can be accessed via [#2928](https://github.com/deepmodeling/abacus-develop/issues/2928)
+
 
 
 ## Advanced Installation Usage

From 2ce0e79b8ae03befbd15b4e5edc90aa1cf3b04da Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 16:37:56 +0800
Subject: [PATCH 15/17] version 2023.5 tag

---
 toolchain/scripts/VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toolchain/scripts/VERSION b/toolchain/scripts/VERSION
index 7668ac0dbaf..219f75857aa 100644
--- a/toolchain/scripts/VERSION
+++ b/toolchain/scripts/VERSION
@@ -1,2 +1,2 @@
 # version file to force a rebuild of the entire toolchain
-VERSION="2023.4"
+VERSION="2023.5"

From b7e663cc597c781f8e1f13a58f0623348eae54d1 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 16:44:50 +0800
Subject: [PATCH 16/17] Update README for GPU version

---
 toolchain/README.md | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index 75a36676e41..08728cafe1a 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -140,7 +140,7 @@ If compliation is successful, a message will be shown like this:
 >     ./build_abacus_intel.sh
 > or you can modify the builder scripts to suit your needs.
 ```
-You can run build_abacus_gnu.sh or build_abacus_intel.sh to build ABACUS 
+You can run `build_abacus_gnu.sh` or `build_abacus_intel.sh` to build ABACUS 
 by gnu-toolchain or intel-toolchain respectively, the builder scripts will
 automatically locate the environment and compile ABACUS.
 You can manually change the builder scripts to suit your needs.
@@ -162,6 +162,18 @@ or you can also do it in a more completely way:
 ```
 
 ## Common Problem and Solution
+### GPU version of ABACUS
+add following options in build*.sh:
+```shell
+cmake -B $BUILD_DIR -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DCMAKE_CXX_COMPILER=icpx \
+        -DMPI_CXX_COMPILER=mpiicpc \
+        ......
+        -DUSE_CUDA=1 \
+        -DCMAKE_CUDA_COMPILER=${path to cuda toolkit}/bin/nvcc \
+        ......
+```
+
 ### shell problem
 If you encounter problem like:
 ```shell

From f6bb704286561e521ccd862c21e5a1fef1b69ea5 Mon Sep 17 00:00:00 2001
From: apple <quanmisaka@stu.pku.edu.cn>
Date: Fri, 10 Nov 2023 16:47:47 +0800
Subject: [PATCH 17/17] update optional message in README

---
 toolchain/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/toolchain/README.md b/toolchain/README.md
index 08728cafe1a..e067fa22c7a 100644
--- a/toolchain/README.md
+++ b/toolchain/README.md
@@ -94,11 +94,12 @@ The needed dependencies version default:
 And Intel-oneAPI need user or server manager to manually install from Intel.
 [Intel-oneAPI](https://www.intel.cn/content/www/cn/zh/developer/tools/oneapi/toolkits.html)
 
-Dependencies below are optional:
+Dependencies below are optional， which is NOT installed by default:
 - `LibTorch` 2.0.1
 - `Libnpy` 0.1.0
 - `LibRI` 0.1.0
 - `LibComm` 0.1.0
+Users can install them by using `--with-*=install` in toolchain*.sh, which is `no` in default.
 
 Notice: for `CEREAL`, `Libnpy`, `LibRI` and `LibComm`, 
 you need to download them from github.com,