diff --git a/.gitignore b/.gitignore index 38038e6e3..cc8a36bda 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ cache .vscode doc/build deploy_logs/ +logs/ +log.* diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 62653a174..24a3773d0 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -277,6 +277,11 @@ modules: environment: set: 'UDUNITS2_XML_PATH': '{prefix}/share/udunits/udunits2.xml' + python: + environment: + set: + 'Python_ROOT_DIR': '{prefix}' + 'Python3_ROOT_DIR': '{prefix}' hierarchy: - mpi diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 48c547702..a3097f320 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -296,3 +296,8 @@ modules: environment: set: 'UDUNITS2_XML_PATH': '{prefix}/share/udunits/udunits2.xml' + python: + environment: + set: + 'Python_ROOT_DIR': '{prefix}' + 'Python3_ROOT_DIR': '{prefix}' diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 069c8c28f..12f225641 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -164,12 +164,14 @@ packages: - '@2.6.4' fms: require: - - '@2025.03' - precision=32,64 - +quad_precision - +openmp - +pic - build_type=Release + - any_of: + - '@=2025.03' + - '@=2024.03' - any_of: - +gfs_phys constants=GFS - ~gfs_phys constants=GEOS diff --git a/configs/common/packages_nag.yaml b/configs/common/packages_nag.yaml new file mode 100644 index 000000000..2273054b5 --- /dev/null +++ b/configs/common/packages_nag.yaml @@ -0,0 +1,31 @@ +packages: + # Compiler preferences + c: + prefer:: + - apple-clang + cxx: + prefer:: + - apple-clang + fortran: + prefer:: + - nag + all: + providers: + fortran: [nag, gcc, llvm, intel-oneapi-compilers] + # Virtual provider settings + blas: + require: + - openblas + fftw-api: + require: + - fftw + lapack: + require: + - openblas + # Individual package settings + ectrans: + require: + - +fftw + oops: + require: + - '~mkl' diff --git a/configs/sites/tier2/macos.gmao/README.md b/configs/sites/tier2/macos.gmao/README.md new file mode 100644 index 000000000..be041f0fa --- /dev/null +++ b/configs/sites/tier2/macos.gmao/README.md @@ -0,0 +1,280 @@ +Default macOS for GMAO + +--- + +## Table of Contents + +- [Overview](#overview) +- [Brew Packages Required](#brew-packages-required) +- [Clone spack-stack](#clone-spack-stack) +- [Setup spack-stack](#setup-spack-stack) +- [Using batch\_install.sh script](#using-batch_installsh-script) + - [First run](#first-run) + - [Subsequent builds](#subsequent-builds) + - [Installing from build caches](#installing-from-build-caches) + - [Loading the stack](#loading-the-stack) +- [Building the stack by hand](#building-the-stack-by-hand) + - [Create Environments](#create-environments) + - [Activate the Environment](#activate-the-environment) + - [Concretize the Environment](#concretize-the-environment) + - [Create Source Cache](#create-source-cache) + - [Pre-Fetch Cargo Dependencies](#pre-fetch-cargo-dependencies) + - [Install Packages](#install-packages) + - [Update Module Files](#update-module-files) + - [Deactivate the Environment](#deactivate-the-environment) +- [Debugging Package Builds](#debugging-package-builds) + +--- + +## Overview + +This configuration provides a generalized Spack Stack setup for GMAO users on macOS using Homebrew. It serves as a generic Tier 2 site `macos.gmao` that dynamically detects your Homebrew installation prefix. + +--- + +## Brew Packages Required + +You must have `brew` installed and the following packages available. (Note: compilers like NAG or Intel are installed out-of-band and configured separately). + +```bash +brew install coreutils +brew install gcc@15 +brew install flang +brew install git +brew install lmod +brew install wget +brew install bash +brew install tcsh +brew install cmake +brew install openssl +brew install rust +``` + +--- + +## Clone spack-stack + +Use the appropriate branch or tag: + +```bash +git clone --recurse-submodules https://github.com/GMAO-SI-Team/spack-stack.git -b geos-testing spack-stack-dev +``` + +--- + +## Setup spack-stack + +Activate the base spack-stack environment: + +```bash +cd spack-stack-dev +. ./setup.sh +``` + +--- + +## Using batch_install.sh script + +The `batch_install.sh` script automates the creation of environments (e.g., `gcc` and `clang` builds), populating your Spack bootstrap caches, and running the Spack installation pipeline. + +### Usage help + +```bash +./util/gmao/batch_install.sh -h +``` + +### Compiler Selection and Auto-Detection + +By default, `batch_install.sh` builds the `gcc@=15.2.0` stack. + +The script will automatically detect your currently active version of Apple Clang (e.g. `apple-clang@21.0.0`) and substitute it into the templates. If the NAG Fortran compiler (`nagfor`) is found in your `PATH`, the script will automatically detect its version as well and add it to the build queue (e.g., `nag@=7.2.7243`). + +If your NAG compiler is installed in a non-standard location and not in your `PATH`, you can explicitly provide its path using the `-N` flag: + +```bash +./util/gmao/batch_install.sh -N /path/to/your/nag/bin/nagfor -r dev -m build -H macos.gmao -e +``` + +If you want to explicitly override the compilers built by the script entirely, use the `-C` flag with a comma-separated list of Spack compiler specs: + +```bash +./util/gmao/batch_install.sh -C "gcc@=15.2.0,nag@=7.2.7243" -r dev -m build -H macos.gmao -e +``` + +*Note: For every compiler you specify, you must have a corresponding `packages_-.yaml.template` file in this site directory.* + +### First run (Building Bootstrap and Source Caches) + +When you first run the script on a new machine, you need to use the `-u` option. This tells Spack to build the bootstrap mirror and source caches locally before attempting to register them. + +```bash +./util/gmao/batch_install.sh -r dev -m build -H macos.gmao -u -e +``` + +- `-r dev -m build`: Sets the developer mode and tells Spack to build the environments. +- `-H macos.gmao`: Overrides hostname detection. This is required to force the script to use this specific generic macOS site, avoiding issues where VPNs or routers mask the real hostname. +- `-u`: Updates and populates the bootstrap/source caches. +- `-e`: Allows continuing builds in existing environments (prevents the script from failing if the environment directories were just created). + +### Subsequent builds + +Once the initial caches are set up, you can run builds without the `-u` flag: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H macos.gmao -e +``` + +### Installing from build caches + +If you are just installing environments using already populated build caches (the typical workflow for users after the initial maintainer setup), use `-m install`: + +```bash +./util/gmao/batch_install.sh -r dev -m install -H macos.gmao -e +``` + +### Generating `.yaml.generated` files +`batch_install.sh` uses `.yaml.template` files in the `macos.gmao` site directory to dynamically detect the path to your Brew installation, your active `apple-clang` version, and your NAG compiler, creating `.yaml.generated` files on the fly. +To prevent cluttering the site configuration directory, the script writes these generated files directly to `configs/sites/tier2/macos.gmao/`, injects them directly into the target environment's `site/` directory during creation, and then automatically cleans up the temporary files from the configurations directory. + +### Loading the stack + +Once the installation and module generation are complete, you can point your shell to the newly built modules. OpenMPI is built specifically with `~two_level_namespace` to support flat namespace linking required by GEOS. + +To ensure executables (like MAPL tests) running under macOS System Integrity Protection (SIP) can properly resolve runtime paths (RPATHs) when using compilers like NAG, the `macos.gmao` site-level `modules.yaml` configuration explicitly permits exporting `DYLD_LIBRARY_PATH` and `DYLD_FALLBACK_LIBRARY_PATH` variables into your environment when loading modules. + +Depending on which compiler you built, you will need to add the correct `install/modulefiles/Core` directory to your `module use` path. + +#### Loading the GCC Stack + +```bash +module use -a /path/to/spack-stack/envs/ge-gcc-15.2.0-build/install/modulefiles/Core +module load stack-gcc stack-openmpi geos-gcm-env +``` + +#### Loading the NAG Stack + +```bash +module use -a /path/to/spack-stack/envs/ge-nag-7.2.7243-build/install/modulefiles/Core +module load stack-nag stack-openmpi geos-gcm-env +``` + +--- + +## Building the stack by hand + +If you prefer to run the Spack commands manually instead of using `batch_install.sh`, you can still follow these steps. However, it's highly recommended to use `batch_install.sh` because it automatically handles dynamic template substitution (e.g., for Homebrew paths, `apple-clang` versions, and NAG compiler versions) that you will otherwise need to do manually with `sed`. + +If you choose to do this manually, you must first run `sed` over the `*.yaml.template` files and create the `.yaml.generated` files. + +### Create Environments + +You only need to create each environment once. Our `macos.gmao` site configurations use `.yaml.template` files which are dynamically processed into `.yaml.generated` files. + +#### GCC Environment + +```bash +spack stack create env --name ge-gcc-15.2.0 --template geos-dev --site macos.gmao --compiler=gcc-15.2.0 +``` + +#### NAG Environment + +```bash +spack stack create env --name genag-nag-7.2.7243 --template geos-dev-nag --site macos.gmao --compiler=nag-7.2.7243 +``` + +--- + +### Activate the Environment + +Navigate to the environment directory you wish to work on. For example, if you built the GCC stack: + +```bash +cd envs/ge-gcc-15.2.0-build +spack env activate -p . +``` + +> **Note:** If you are building or debugging a different compiler stack, be sure to `cd` into that specific environment directory instead (e.g., `envs/ge-nag-7.2.7243-build` or `envs/ge-clang-22.1.3-build`). + +> **Important:** Run this in *every* terminal where you plan to run Spack commands for this environment. + +--- + +### Concretize the Environment + +```bash +spack concretize 2>&1 | tee log.concretize ; bell +``` + +*(Optional `bell` helper: `bell() { tput bel ; printf "\nFinished at: " ; date; }`)* + +--- + +### Create Source Cache + +This downloads all source tarballs for your concretized environment to prevent network timeouts during the build: + +```bash +spack mirror create -a -d $HOME/spack-stack-mirrors/spack-source-mirror +``` + +> ⚠️ **Do not run this outside an activated environment.** +> Otherwise, Spack will attempt to mirror **every** known package and version in the registry. + +--- + +### Pre-Fetch Cargo Dependencies + +Rust packages frequently require network access during the build. Pre-fetch their dependencies to the cargo mirror: + +```bash +export CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror +../../util/fetch_cargo_deps.py +``` + +> ⚠️ **Set `CARGO_HOME` in your environment before running `spack install`.** + +--- + +### Install Packages + +```bash +export CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror +spack install -j 6 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell +``` + +> **Note:** You may need to re-run this command if a package fails due to an intermittent network issue or parallel build race condition. + +--- + +### Update Module Files + +After installation completes, regenerate the Lmod module tree and meta-modules: + +```bash +spack module lmod refresh -y --delete-tree ; bell +spack stack setup-meta-modules +``` + +--- + +### Deactivate the Environment + +```bash +spack env deactivate +``` + +--- + +## Debugging Package Builds + +If a specific package fails to build, you can drop into Spack's build environment to debug it manually: + +```bash +spack clean +spack stage +cd $(spack location -s ) +spack build-env -- bash --norc --noprofile +``` + +This drops you into a clean bash shell with the exact environment variables, compiler wrappers, and dependencies loaded that Spack uses during the build. + diff --git a/configs/sites/tier2/macos.gmao/config.yaml b/configs/sites/tier2/macos.gmao/config.yaml new file mode 100644 index 000000000..8ec652ba9 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/config.yaml @@ -0,0 +1,3 @@ +config: + build_jobs: 6 + installer: old diff --git a/configs/sites/tier2/macos.gmao/mirrors.yaml.template b/configs/sites/tier2/macos.gmao/mirrors.yaml.template new file mode 100644 index 000000000..f2e285635 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/mirrors.yaml.template @@ -0,0 +1,3 @@ +mirrors: + local-source: file://@HOME@/spack-stack-mirrors/spack-source-mirror + local-binary: file://@HOME@/spack-stack-mirrors/spack-build-mirror diff --git a/configs/sites/tier2/macos.gmao/modules.yaml b/configs/sites/tier2/macos.gmao/modules.yaml new file mode 100644 index 000000000..320999839 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/modules.yaml @@ -0,0 +1,41 @@ +modules: + default: + enable:: + - lmod + prefix_inspections:: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH + share/pkgconfig: + - PKG_CONFIG_PATH + '': + - CMAKE_PREFIX_PATH + lmod: + openmpi: + environment: + set: + 'TMPDIR': '/tmp' + 'OMPI_MCA_btl': '^tcp' + all: + filter: + exclude_env_vars: + - LD_LIBRARY_PATH + include: + # Compiler modules + - apple-clang + - gcc + - llvm + # MPI modules + - mpich + - openmpi diff --git a/configs/sites/tier2/macos.gmao/packages.yaml b/configs/sites/tier2/macos.gmao/packages.yaml new file mode 100644 index 000000000..57307fcb5 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/packages.yaml @@ -0,0 +1,26 @@ +packages: + + netcdf-c: + require: [+shared] + + openblas: + require: + - '~dynamic_dispatch' + - 'build_system=cmake' + + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + git: + externals: + - spec: git@2.50.1~tcltk + prefix: /usr + openssh: + externals: + - spec: openssh@9.9p2 + prefix: /usr + perl: + externals: + - spec: perl@5.34.1~cpanm+opcode+open+shared+threads + prefix: /usr diff --git a/configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template b/configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template new file mode 100644 index 000000000..49fa8d5ef --- /dev/null +++ b/configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template @@ -0,0 +1,39 @@ +packages: + c: + require: [apple-clang@@APPLE_CLANG_VERSION@] + cxx: + require: [apple-clang@@APPLE_CLANG_VERSION@] + fortran: + require: [llvm@21.1.3 ~clang +flang] + mpi: + require: [openmpi@=5.0.10] + apple-clang: + externals: + - spec: apple-clang@@APPLE_CLANG_VERSION@ languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + llvm: + buildable: False + externals: + - spec: llvm@21.1.3 ~clang +flang languages:='fortran' + prefix: @BREW_PREFIX@ + extra_attributes: + compilers: + fortran: @BREW_PREFIX@/bin/flang + + openmpi: + require:: + - '+internal-hwloc' + - '+internal-libevent' + - '+internal-pmix' + - '+fortran' + # GEOS requires a flat namespace to link properly on macOS + - '~two_level_namespace' + + gdbm: + externals: + - spec: gdbm@1.26 + prefix: @BREW_PREFIX@/opt/gdbm diff --git a/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template b/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template new file mode 100644 index 000000000..20c3ac212 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template @@ -0,0 +1,48 @@ +packages: + c: + require: [apple-clang@@APPLE_CLANG_VERSION@] + cxx: + require: [apple-clang@@APPLE_CLANG_VERSION@] + fortran: + require: [gcc@15.2.0] + mpi: + require: [openmpi@=5.0.10] + apple-clang: + externals: + - spec: apple-clang@@APPLE_CLANG_VERSION@ languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + gcc: + buildable: False + externals: + - spec: gcc@15.2.0 languages:='fortran' + prefix: @BREW_PREFIX@ + extra_attributes: + compilers: + fortran: @BREW_PREFIX@/bin/gfortran-15 + + openmpi: + require:: + - '+internal-hwloc' + - '+internal-libevent' + - '+internal-pmix' + - '+fortran' + # GEOS requires a flat namespace to link properly on macOS + - '~two_level_namespace' + + # ESMF by default for Darwin.gfortranclang turns off OpenMP, so + # we do the same here to avoid build failures. + # We also build as +debug only because testing shows that (so far) + # esmf + Darwin + gfortranclang needs ESMF_BOPT=g to work with GEOS + esmf: + require: + - '~openmp' + - '+debug' + + gdbm: + externals: + - spec: gdbm@1.26 + prefix: @BREW_PREFIX@/opt/gdbm diff --git a/configs/sites/tier2/macos.gmao/packages_nag.yaml.template b/configs/sites/tier2/macos.gmao/packages_nag.yaml.template new file mode 100644 index 000000000..6a6a024af --- /dev/null +++ b/configs/sites/tier2/macos.gmao/packages_nag.yaml.template @@ -0,0 +1,61 @@ +packages: + c: + require: [apple-clang@@APPLE_CLANG_VERSION@] + cxx: + require: [apple-clang@@APPLE_CLANG_VERSION@] + fortran: + require: [nag@@NAG_VERSION@] + mpi: + require: [openmpi@=5.0.7] + apple-clang: + externals: + - spec: apple-clang@@APPLE_CLANG_VERSION@ languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + nag: + buildable: False + externals: + - spec: nag@@NAG_VERSION@ languages:='fortran' + prefix: @NAG_PREFIX@ + extra_attributes: + compilers: + fortran: @NAG_PATH@ + + openmpi: + require:: + - '@=5.0.7' + - '+internal-hwloc' + - '+internal-libevent' + - '+internal-pmix' + - '+fortran' + # GEOS requires a flat namespace to link properly on macOS + - '~two_level_namespace' + + # ESMF by default for Darwin.gfortranclang turns off OpenMP, so + # we do the same here to avoid build failures. + # Also enforce +debug since NAG is meant for debugging. + esmf: + require:: + - '~openmp' + - '+debug' + + # Disable parallel-netcdf for NAG as it fails to build and is unneeded + parallelio: + require:: + - '@2.6.2' + - '~pnetcdf' + + # netcdf-fortran needs explicit preprocessor and type safety flags for NAG + netcdf-fortran: + require:: + - 'fflags="-fpp -mismatch_all"' + + + gdbm: + externals: + - spec: gdbm@1.26 + prefix: @BREW_PREFIX@/opt/gdbm + diff --git a/configs/templates/geos-dev-nag/spack.yaml b/configs/templates/geos-dev-nag/spack.yaml new file mode 100644 index 000000000..a61c1007d --- /dev/null +++ b/configs/templates/geos-dev-nag/spack.yaml @@ -0,0 +1,23 @@ +# Template for GEOS development +# +spack: + concretizer: + unify: when_possible + + view: false + include: [] + + specs: + # NOTE: Do not explicitly add ~debug to the first geos-gcm-env spec below. + # On some platforms (like macOS with gcc-15/gfortranclang), ESMF requires + # +debug to build successfully, which is enforced via the site's packages.yaml. + # Explicitly requesting ~debug here will cause a concretization conflict on those platforms. + - geos-gcm-env ^esmf@=9.0.0b11 + - geos-gcm-env +debug ^esmf@=9.0.0b11 + + + packages: + # Turn on python variant for esmf + esmf: + require: + - +python diff --git a/configs/templates/geos-dev/spack.yaml b/configs/templates/geos-dev/spack.yaml new file mode 100644 index 000000000..0e317ec2f --- /dev/null +++ b/configs/templates/geos-dev/spack.yaml @@ -0,0 +1,25 @@ +# Template for GEOS development +# +spack: + concretizer: + unify: when_possible + + view: false + include: [] + + specs: + # NOTE: Do not explicitly add ~debug to the first geos-gcm-env spec below. + # On some platforms (like macOS with gcc-15/gfortranclang), ESMF requires + # +debug to build successfully, which is enforced via the site's packages.yaml. + # Explicitly requesting ~debug here will cause a concretization conflict on those platforms. + - geos-gcm-env ^esmf@=9.0.0b11 + - geos-gcm-env +debug ^esmf@=9.0.0b11 + + # Various fms builds + - fms@=2024.03 ~gfs_phys constants=GEOS + + packages: + # Turn on python variant for esmf + esmf: + require: + - +python diff --git a/repos/builtin b/repos/builtin index 89af8e824..e667ed19d 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 89af8e82400c8b5b8ae62c2915e7707cb4b8445f +Subproject commit e667ed19dfc05e5ba15c5e80f718993dc47d5abc diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py index 47c5ff19f..91576e9e5 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2026 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -18,11 +18,17 @@ class GeosGcmEnv(BundlePackage): # Current version version("1.1.0") + variant("debug", default=False, description="Build debug version of selected dependencies") + depends_on("base-env", type="run") + depends_on("blas", type="run") depends_on("lapack", type="run") + depends_on("mepo", type="run") - depends_on("esmf", type="run") + depends_on("esmf ~debug", type="run", when="~debug") + depends_on("esmf +debug", type="run", when="+debug") + # mapl is built as part of GEOS, don't load; # needs external gftl-shared/fargparse/pflogger # depends_on("mapl", type="run") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 447891dc9..5d582aa55 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -403,10 +403,10 @@ def custom_sort_key(entry): logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] - substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] - substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] - substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] + substitutes["CC"] = compiler.extra_attributes.get("compilers", {}).get("c", "") + substitutes["CXX"] = compiler.extra_attributes.get("compilers", {}).get("cxx", "") + substitutes["F77"] = compiler.extra_attributes.get("compilers", {}).get("fortran", "") + substitutes["FC"] = compiler.extra_attributes.get("compilers", {}).get("fortran", "") logging.debug(" ... ... CC : {}".format(substitutes["CC"])) logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh new file mode 100755 index 000000000..c00450964 --- /dev/null +++ b/util/gmao/batch_install.sh @@ -0,0 +1,922 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SPACK_STACK_DIR=$(dirname $(dirname ${SCRIPT_DIR})) + +set -e + +################################################################################################## +# Packages for which to run tests when "-t" is specified; caveat: must be listed in order of # +# their respective dependencies (e.g. A depends on B --> B comes first) # +################################################################################################## + +SPACK_STACK_PACKAGES_TO_TEST=( + "oops" + "ioda" + "ioda-converters" + "ropp-ufo" + "ufo" +) + +################################################################################################## +# Options # +################################################################################################## + +################################################################################################## +# macOS Prerequisites Check # +################################################################################################## + +check_macos_prerequisites() { + if ! command -v brew &> /dev/null; then + echo "ERROR: brew is not installed or not in PATH." + exit 1 + fi + + local missing_pkgs=() + local required_pkgs=(coreutils gcc git lmod wget bash tcsh cmake openssl rust) + + for pkg in "${required_pkgs[@]}"; do + if ! brew --prefix "$pkg" &> /dev/null; then + missing_pkgs+=("$pkg") + fi + done + + if [ ${#missing_pkgs[@]} -ne 0 ]; then + echo "ERROR: Missing required Homebrew packages: ${missing_pkgs[*]}" + echo "Please run: brew install ${missing_pkgs[*]}" + exit 1 + fi +} + + +usage() { + set +x + echo + echo "Usage: $0 -r -m [-d ] [-c ] [-H ]" + echo + echo " -r Set role, can be 'ops' or 'dev'" + echo " -m Set mode, can be 'build' or 'install';" + echo " build: build environments and update build caches;" + echo " install: install environments using build caches" + echo " -d Build or install environments in ENV_DIRS;" + echo " if not set, the default location is used" + echo " -c Provide location of build caches as BUILDCACHE_DIR;" + echo " if not set, authoritative build caches are used" + echo " -u Flag to update bootstrap and source caches;" + echo " requires role 'dev' and mode 'build'" + echo " -e Continue builds/install in existing environments;" + echo " by default, exit with an error if already exist" + echo " -C Set a comma-separated list of compilers to use (e.g. gcc@=15.2.0,nag@=7.2.7243);" + echo " overrides the default compilers for the site" + echo " -N Path to nagfor executable (e.g. /opt/nag/bin/nagfor);" + echo " forces NAG stack to be built using this specific compiler" + echo " -s Submit 'spack install' to batch scheduler" + echo " -t Run tests for specific thirdparty dependencies;" + echo " these are currently hardcoded in batch_install.sh" + echo " -n Dry-run: print what would be executed without running anything" + echo " -H Provide hostname manually (overrides autodetection);" + echo " useful when VPN/etc masks the real hostname" + echo " -h display this help" + echo +} + +while getopts r:m:d:c:C:N:H:nuesth flag +do + case "${flag}" in + r) + SPACK_STACK_ROLE=${OPTARG} + ;; + m) + SPACK_STACK_MODE=${OPTARG} + ;; + d) + SPACK_STACK_ENVIRONMENT_DIRS=$(readlink -f ${OPTARG}) + ;; + c) + SPACK_STACK_BUILDCACHE_DIR=$(readlink -f ${OPTARG}) + ;; + C) + SPACK_STACK_COMPILER_OPT=${OPTARG} + ;; + N) + SPACK_STACK_NAGFOR_PATH=${OPTARG} + ;; + H) + SPACK_STACK_BATCH_HOST_OPT=${OPTARG} + ;; + n) + SPACK_STACK_DRY_RUN="true" + ;; + u) + SPACK_STACK_UPDATE_DEV_CACHES="true" + ;; + e) + SPACK_STACK_IGNORE_ENV_EXIST="true" + ;; + s) + SPACK_STACK_SUBMIT_TO_SCHEDULER="true" + ;; + t) + SPACK_STACK_RUN_TESTS="true" + ;; + *) + usage + exit 1 + ;; + esac +done + +echo "INFO: $0 options:" +echo " SPACK_STACK_ROLE: ${SPACK_STACK_ROLE:-not set}" +echo " SPACK_STACK_MODE: ${SPACK_STACK_MODE:-not set}" +echo " SPACK_STACK_ENVIRONMENT_DIRS: ${SPACK_STACK_ENVIRONMENT_DIRS:-${SPACK_STACK_DIR}/envs}" +echo " SPACK_STACK_BUILDCACHE_DIR: ${SPACK_STACK_BUILDCACHE_DIR:-use default caches}" +echo " SPACK_STACK_BATCH_HOST_OPT: ${SPACK_STACK_BATCH_HOST_OPT:-autodetect}" +echo " SPACK_STACK_DRY_RUN: ${SPACK_STACK_DRY_RUN:-false}" +echo " SPACK_STACK_UPDATE_DEV_CACHES: ${SPACK_STACK_UPDATE_DEV_CACHES:-false}" +echo " SPACK_STACK_IGNORE_ENV_EXIST: ${SPACK_STACK_IGNORE_ENV_EXIST:-false}" +echo " SPACK_STACK_SUBMIT_TO_SCHEDULER: ${SPACK_STACK_SUBMIT_TO_SCHEDULER:-false}" +echo " SPACK_STACK_RUN_TESTS: ${SPACK_STACK_RUN_TESTS:-false}" + +if [[ -z ${SPACK_STACK_ROLE} ]]; then + echo "ERROR, SPACK_STACK_ROLE not defined. Provide -r ROLE as argument" + exit 1 +elif [[ ! ${SPACK_STACK_ROLE} == "dev" && ! ${SPACK_STACK_ROLE} == "ops" ]]; then + echo "ERROR, invalid role '${SPACK_STACK_ROLE}'" + exit 1 +fi + +if [[ -z ${SPACK_STACK_MODE} ]]; then + echo "ERROR, SPACK_STACK_MODE not defined. Provide -m MODE as argument" + exit 1 +elif [[ ! ${SPACK_STACK_MODE} == "build" && ! ${SPACK_STACK_MODE} == "install" ]]; then + echo "ERROR, invalid mode '${SPACK_STACK_MODE}'" + exit 1 +fi + +# Role ops cannot write to the default (authoritative) build cache +if [[ ${SPACK_STACK_ROLE} == "ops" && ${SPACK_STACK_MODE} == "build" && -z ${SPACK_STACK_BUILDCACHE_DIR} ]]; then + echo "ERROR, SPACK_STACK_BUILDCACHE_DIR not defined. Provide -c BUILDCACHE_DIR" + echo "as argument when role is 'ops' and mode is 'build'" + exit 1 +fi + +# Updating bootstrap and source caches requires role dev and mode build +if [[ ${SPACK_STACK_UPDATE_DEV_CACHES} == "true" ]]; then + if [[ ! ${SPACK_STACK_ROLE} == "dev" || ! ${SPACK_STACK_MODE} == "build" ]]; then + echo "ERROR, SPACK_STACK_UPDATE_DEV_CACHES requires role 'dev' and mode 'build'" + exit 1 + fi +fi + +################################################################################################## + +if [[ -n "${SPACK_STACK_BATCH_HOST_OPT}" ]]; then + SPACK_STACK_BATCH_HOST="${SPACK_STACK_BATCH_HOST_OPT}" +else + # Remove domain name suffices and digits to determine hostname + SPACK_STACK_BATCH_HOST=$(echo ${HOSTNAME} | cut -d "." -f 1) + SPACK_STACK_BATCH_HOST=${SPACK_STACK_BATCH_HOST//[0-9]/} +fi + +case ${SPACK_STACK_BATCH_HOST} in + nas) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=13.2.0") + SPACK_STACK_BATCH_TEMPLATES=("unified-dev") + SPACK_STACK_MODULE_CHOICE="tcl" + SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + ;; + nas-toss5) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.1") + SPACK_STACK_BATCH_TEMPLATES=("unified-dev") + SPACK_STACK_MODULE_CHOICE="tcl" + SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + ;; + discover-gmao) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.1") + SPACK_STACK_BATCH_TEMPLATES=("unified-dev") + SPACK_STACK_MODULE_CHOICE="lmod" + SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + ;; + macos.gmao) + # Detect NAG Fortran Compiler + nag_path_tmp="" + if [[ -n "${SPACK_STACK_NAGFOR_PATH}" && -x "${SPACK_STACK_NAGFOR_PATH}" ]]; then + nag_path_tmp="${SPACK_STACK_NAGFOR_PATH}" + elif command -v nagfor &> /dev/null; then + nag_path_tmp=$(which nagfor) + fi + + if [[ -n "${nag_path_tmp}" ]]; then + export MAC_GMAO_NAG_PATH="${nag_path_tmp}" + export MAC_GMAO_NAG_VERSION=$("${MAC_GMAO_NAG_PATH}" -V 2>&1 | head -n1 | sed -E 's/.*Release ([0-9]+\.[0-9]+).*Build ([0-9]+).*/\1.\2/' || echo "7.2.7243") + export MAC_GMAO_NAG_PREFIX=$(dirname $(dirname "${MAC_GMAO_NAG_PATH}")) + fi + + if [[ -n "${SPACK_STACK_COMPILER_OPT}" ]]; then + IFS=',' read -r -a SPACK_STACK_BATCH_COMPILERS <<< "${SPACK_STACK_COMPILER_OPT}" + else + SPACK_STACK_BATCH_COMPILERS=("gcc@=15.2.0") + if [[ -n "${MAC_GMAO_NAG_VERSION}" ]]; then + SPACK_STACK_BATCH_COMPILERS+=("nag@=${MAC_GMAO_NAG_VERSION}") + fi + fi + + # Auto-detect Apple Clang version + if command -v clang &> /dev/null; then + export MAC_GMAO_APPLE_CLANG_VERSION=$(clang --version | grep "Apple clang version" | awk '{print $4}') + else + export MAC_GMAO_APPLE_CLANG_VERSION="21.0.0" + fi + + SPACK_STACK_BATCH_TEMPLATES=("geos-dev" "geos-dev-nag") + SPACK_STACK_MODULE_CHOICE="lmod" + SPACK_STACK_BOOTSTRAP_MIRROR="${HOME}/spack-stack-mirrors/spack-bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="${HOME}/spack-stack-mirrors/spack-cargo-mirror" + ;; + *) + echo "ERROR, host ${SPACK_STACK_BATCH_HOST} not configured" + exit 1 + ;; +esac + +################################################################################################## + +function fix_permissions() { + host=$1 + dir=$2 + executables=$3 + echo "Repairing permissions for directory ${dir} on ${host} ..." + set +e + case ${host} in + nas) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; + nas-toss5) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; + discover-gmao) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; + macos.gmao) + ;; + *) + echo "ERROR, xargs-chmod command not configured for ${host}" + exit 1 + ;; + esac + set -e +} + +################################################################################################## + +function tasks_per_node() { + host=$1 + case ${host} in + nas) + tpn=128 + ;; + nas-toss5) + tpn=256 + ;; + discover-gmao) + tpn=128 + ;; + *) + echo "ERROR, tasks_per_node command not configured for ${host}" + exit 1 + ;; + esac + echo "${tpn}" +} + +################################################################################################## + +function run_interactive_job() { + host=$1 + script=$2 + reuse_build_cache=$3 + tpn=$(tasks_per_node ${host}) + if [[ "${reuse_build_cache}" == "true" ]]; then + walltime="120" + else + walltime="720" + fi + if [[ ! -n "${ACCOUNT}" ]]; then + echo "ERROR, environment variable ACCOUNT not set" + exit 1 + fi + echo "Starting interactive job on ${host} with ${tpn} tasks and a walltime of ${walltime} minutes for ${script} ..." + case ${host} in + nas) + module load slurm + salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} bash ${script} + module unload slurm + ;; + nas-toss5) + salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} --qos=serial --account=${ACCOUNT} bash ${script} + ;; + discover-gmao) + salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} --qos=compute --account=${ACCOUNT} bash ${script} + ;; + *) + echo "ERROR, run_interactive_job command not configured for ${host}" + exit 1 + ;; + esac +} + +################################################################################################## + +echo +echo "Welcome to GMAO SPACK-STACK BATCH INSTALL" +echo + +LOG_TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +mkdir -p "${SPACK_STACK_DIR}/logs" + +if [[ ! -e "setup.sh" || ! -e ".spackstack" ]]; then + echo "ERROR, this script must be executed from the top-level spack-stack directory" + exit 1 +fi + +host=${SPACK_STACK_BATCH_HOST} +module_choice=${SPACK_STACK_MODULE_CHOICE} +bootstrap_mirror_path=${SPACK_STACK_BOOTSTRAP_MIRROR} +cargo_mirror_path=${SPACK_STACK_CARGO_MIRROR} +export CARGO_HOME=${cargo_mirror_path} + +if [[ -z ${SPACK_STACK_ENVIRONMENT_DIRS} ]]; then + environment_dirs=${PWD}/envs +else + environment_dirs=${SPACK_STACK_ENVIRONMENT_DIRS} +fi +[[ "${SPACK_STACK_DRY_RUN}" != "true" ]] && mkdir -p ${environment_dirs} + +if [[ ! -z ${SPACK_STACK_BUILDCACHE_DIR} ]]; then + buildcache_dir=${SPACK_STACK_BUILDCACHE_DIR} + if [[ "${SPACK_STACK_MODE}" == "install" && ! -d ${buildcache_dir} ]]; then + echo "ERROR, build cache ${buildcache_dir} not found," + echo "must exist before installing environments" + exit 1 + else + [[ "${SPACK_STACK_DRY_RUN}" != "true" ]] && mkdir -p ${buildcache_dir} + fi +fi + +if [[ "${SPACK_STACK_MODE}" == "install" ]]; then + update_bootstrap_mirror="false" + update_cargo_mirror="false" + update_source_cache="false" + update_build_cache="false" + reuse_build_cache="true" +elif [[ "${SPACK_STACK_MODE}" == "build" ]]; then + if [[ "${SPACK_STACK_ROLE}" == "ops" ]]; then + update_bootstrap_mirror="false" + update_cargo_mirror="false" + update_source_cache="false" + elif [[ "${SPACK_STACK_ROLE}" == "dev" ]]; then + if [[ ${SPACK_STACK_UPDATE_DEV_CACHES} == "true" ]]; then + update_bootstrap_mirror="true" + update_cargo_mirror="true" + update_source_cache="true" + else + update_bootstrap_mirror="false" + update_cargo_mirror="false" + update_source_cache="false" + fi + else + echo "ERROR, invalid role ${SPACK_STACK_ROLE}" + exit 1 + fi + update_build_cache="true" + reuse_build_cache="true" +else + echo "ERROR, invalid mode ${SPACK_STACK_MODE}" + exit 1 +fi + +ignore_env_exist=${SPACK_STACK_IGNORE_ENV_EXIST:-false} + +if [[ "${SPACK_STACK_SUBMIT_TO_SCHEDULER}" == "true" ]]; then + submit_to_scheduler="true" +else + submit_to_scheduler="false" +fi + +if [[ "${SPACK_STACK_RUN_TESTS}" == "true" ]]; then + test_packages=("${SPACK_STACK_PACKAGES_TO_TEST[@]}") +else + test_packages=() +fi + +# Loop through all compilers and templates for this host +first_pass="true" +for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do + + if [[ ! ${compiler} == *"@="* ]]; then + echo "ERROR, '@=' not found in compiler string '${compiler}'" + exit 1 + fi + + compiler_name=$(echo ${compiler} | cut -d "@" -f 1) + compiler_version=$(echo ${compiler} | cut -d "=" -f 2) + + for template in "${SPACK_STACK_BATCH_TEMPLATES[@]}"; do + + echo + ############################################################# + # Add excluded combinations of compilers and templates here # + ############################################################# + # cylc-dev only with gcc + #if [[ "${template}" == "cylc-dev" && ! "${compiler_name}" == "gcc" ]]; then + #echo "Skipping template ${template} with compiler ${compiler}" + #continue + ## With clang, only neptune-dev-llvm + #elif [[ "${compiler_name}" == "clang" && ! "${template}" == "neptune-dev-llvm" ]]; then + #echo "Skipping template ${template} with compiler ${compiler}" + #continue + ## With other compilers, skip neptune-dev-llvm + #elif [[ ! "${compiler_name}" == "clang" && "${template}" == "neptune-dev-llvm" ]]; then + #echo "Skipping template ${template} with compiler ${compiler}" + #continue + ## FMS compiler ICE: https://github.com/NOAA-GFDL/FMS/issues/1680 + #elif [[ "${compiler_name}" == "oneapi" && "${compiler_version}" == "2025.1"* && "${template}" == "unified-dev" ]]; then + #echo "Skipping template ${template} with compiler ${compiler}" + #continue + #fi + if [[ "${template}" == "geos-dev" && "${compiler_name}" == "nag" ]]; then + echo "Skipping template ${template} with compiler ${compiler} (fms not supported by nag)" + continue + elif [[ "${template}" == "geos-dev-nag" && "${compiler_name}" != "nag" ]]; then + echo "Skipping template ${template} with compiler ${compiler} (geos-dev-nag is only for nag)" + continue + fi + echo "Processing template ${template} with compiler ${compiler}" + ############################################################# + + # Build environment name. Prefices are defined here + case ${template} in + unified-dev) + env_name_prefix="ue" + ;; + geos-dev) + env_name_prefix="ge" + ;; + geos-dev-nag) + env_name_prefix="ge" + ;; + *) + echo "ERROR, template ${template} not configured" + exit 1 + ;; + esac + env_name=${env_name_prefix}-${compiler_name}-${compiler_version} + [[ "${update_build_cache}" == "true" ]] && env_name=${env_name}-build + env_dir=${environment_dirs}/${env_name} + + # Reset env_exists for this specific environment target + env_exists="false" + + # Bail out if the environment already exists + if [[ -d ${env_dir} ]]; then + if [[ ${ignore_env_exist} == "true" ]]; then + env_exists="true" + else + if [[ "${SPACK_STACK_DRY_RUN}" == "true" ]]; then + echo "[DRY-RUN] ERROR: environment ${env_dir} already exists. (Would exit here)" + continue + else + echo "ERROR, environment ${env_dir} already exists" + exit 1 + fi + fi + fi + + if [[ "${SPACK_STACK_DRY_RUN}" == "true" ]]; then + echo "--------------------------------------------------------------------------------" + echo "[DRY-RUN] Target Environment: ${env_name}" + echo "[DRY-RUN] Directory: ${env_dir}" + echo "--------------------------------------------------------------------------------" + if [[ "${update_bootstrap_mirror}" == "true"* ]]; then + echo "[DRY-RUN] spack bootstrap mirror --binary-packages ${PWD}/tmp-bootstrap-mirror" + echo "[DRY-RUN] rsync -a ${PWD}/tmp-bootstrap-mirror/ ${bootstrap_mirror_path}/" + echo "[DRY-RUN] spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache" + update_bootstrap_mirror="false" + fi + + if [[ ! ${env_exists} == "true" ]]; then + if [[ "${host}" == "macos.gmao" ]]; then + echo "[DRY-RUN] Would check macOS prerequisites and generate YAML configs from templates" + fi + echo "[DRY-RUN] spack stack create env --name=${env_name} \\" + echo " --site=${host} --compiler=${compiler_name}-${compiler_version} \\" + echo " --template=${template} --dir=${environment_dirs} --treat-warnings-as-errors" + fi + echo "[DRY-RUN] spack env activate -p ${env_dir}" + if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then + echo "[DRY-RUN] spack external find --not-buildable autoconf automake bash cmake cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo" + echo "[DRY-RUN] generating spack-macos-externals.yaml and applying with 'spack config add -f'" + fi + echo "[DRY-RUN] spack bootstrap now" + echo "[DRY-RUN] spack concretize --force --fresh" + + if [[ "${update_source_cache}" == "true"* ]]; then + echo "[DRY-RUN] spack mirror create -a -d " + fi + if [[ "${update_cargo_mirror}" == "true"* ]]; then + echo "[DRY-RUN] ./util/fetch_cargo_deps.py" + fi + + echo "[DRY-RUN] Generating spack-install.${env_name}.sh and executing via:" + if [[ "${submit_to_scheduler}" == "true" ]]; then + echo "[DRY-RUN] run_interactive_job ${host} spack-install.${env_name}.sh ${reuse_build_cache}" + else + echo "[DRY-RUN] bash spack-install.${env_name}.sh" + fi + + if [[ "${update_build_cache}" == "true" ]]; then + echo "[DRY-RUN] spack buildcache push -u " + echo "[DRY-RUN] spack buildcache update-index local-binary" + else + echo "[DRY-RUN] spack module ${module_choice} refresh --yes --upstream-modules" + echo "[DRY-RUN] spack stack setup-meta-modules" + fi + + echo "[DRY-RUN] spack clean -d -f -m -p -s" + echo "[DRY-RUN] spack env deactivate" + echo "" + first_pass="false" + continue + fi + + # Reset environment + echo "Resetting environment ..." + case ${host} in + atlantis) + umask 0022 + module purge + case ${compiler} in + clang@=22.1.0) + module use /gpfs/neptune/spack-stack/llvm-22.1.0/modulefiles + module use /gpfs/neptune/spack-stack/openmpi-4.1.8/llvm-22.1.0/modulefiles + ;; + gcc@=13.4.0) + module use /gpfs/neptune/spack-stack/gcc-13.4.0/modulefiles + module use /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles + ;; + oneapi@=2025.3.0) + module use /gpfs/neptune/spack-stack/oneapi-2025.3.0/modulefiles + ;; + esac + ;; + nas) + umask 0022 + set +e + module purge + set -e + ;; + nas-toss5) + umask 0022 + set +e + module purge + set -e + ;; + discover-gmao) + umask 0022 + set +e + module purge + set -e + ;; + macos.gmao) + set +e + ulimit -s unlimited 2>/dev/null || ulimit -s hard 2>/dev/null || ulimit -s 65532 2>/dev/null || true + if ! command -v module &> /dev/null; then + if command -v brew &> /dev/null; then + . $(brew --prefix)/opt/lmod/init/bash 2>/dev/null || true + fi + fi + set -e + ;; + *) + echo "ERROR, host ${host} not configured for resetting environment" + exit 1 + ;; + esac + + # Info prints + ulimit -a + module li + + source setup.sh + if [[ "${first_pass}" == "true" ]]; then + spack clean -a + else + # Don't remove software and configuration needed to bootstrap Spack + spack clean -d -f -m -p -s + fi + + # Update bootstrap mirror if requested before creating any + # environments. It is sufficient to do this one time only. + if [[ "${update_bootstrap_mirror}" == "true"* ]]; then + tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror + echo "Creating bootstrap mirror ${tmp_bootstrap_mirror_path} ..." + rm -fr ${tmp_bootstrap_mirror_path} + spack bootstrap mirror --binary-packages ${tmp_bootstrap_mirror_path} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.bootstrap-mirror.${LOG_TIMESTAMP} + rsync -a ${tmp_bootstrap_mirror_path}/ ${bootstrap_mirror_path}/ + rm -fr ${tmp_bootstrap_mirror_path} + # Update buildcache index + spack buildcache update-index ${bootstrap_mirror_path}/bootstrap_cache + # Fix permissions for the bootstrap mirror + fix_permissions ${host} ${bootstrap_mirror_path} 0 + update_bootstrap_mirror="false" + # When spack creates a bootstrap mirror, it populates the "spack" scope + # with compilers and packages it finds, which can create problems later + echo "Removing package config in spack/etc/spack created by spack boostrap mirror" + rm -vf spack/etc/spack/packages.yaml + fi + + if [[ ! ${env_exists} == "true" ]]; then + if [[ "${host}" == "macos.gmao" ]]; then + check_macos_prerequisites + + macos_site_dir="${SPACK_STACK_DIR}/configs/sites/tier2/macos.gmao" + brew_prefix=$(brew --prefix) + + # Use NAG vars if available + nag_version=${MAC_GMAO_NAG_VERSION} + nag_path=${MAC_GMAO_NAG_PATH} + nag_prefix=${MAC_GMAO_NAG_PREFIX} + + apple_clang_version=${MAC_GMAO_APPLE_CLANG_VERSION:-"21.0.0"} + + for template_file in "${macos_site_dir}"/*.yaml.template; do + if [[ -f "${template_file}" ]]; then + filename=$(basename "${template_file}") + base_filename="${filename%.template}" + + # Special case for NAG template: inject version into filename + if [[ "${base_filename}" == "packages_nag.yaml" && -n "${nag_version}" ]]; then + base_filename="packages_nag-${nag_version}.yaml" + fi + + sed_cmd="sed -e \"s#@HOME@#${HOME}#g\" -e \"s#@BREW_PREFIX@#${brew_prefix}#g\" -e \"s#@APPLE_CLANG_VERSION@#${apple_clang_version}#g\"" + if [[ -n "${nag_version}" ]]; then + sed_cmd="${sed_cmd} -e \"s#@NAG_VERSION@#${nag_version}#g\" -e \"s#@NAG_PREFIX@#${nag_prefix}#g\" -e \"s#@NAG_PATH@#${nag_path}#g\"" + fi + + eval "${sed_cmd} \"${template_file}\"" > "${SPACK_STACK_DIR}/configs/sites/tier2/${host}/${base_filename}" + if [[ -d "${SPACK_STACK_DIR}/.git" ]]; then + grep -q "^configs/sites/tier2/${host}/${base_filename}$" "${SPACK_STACK_DIR}/.git/info/exclude" 2>/dev/null || echo "configs/sites/tier2/${host}/${base_filename}" >> "${SPACK_STACK_DIR}/.git/info/exclude" + fi + fi + done + fi + + spack stack create env --name=${env_name} \ + --site=${host} \ + --compiler=${compiler_name}-${compiler_version} \ + --template=${template} \ + --dir=${environment_dirs} \ + --treat-warnings-as-errors \ + 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.create.${env_name}.${LOG_TIMESTAMP} + + # Clean up the generated yamls in the site configuration now that the env is created + if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then + rm -f "${SPACK_STACK_DIR}/configs/sites/tier2/${host}/mirrors.yaml" + rm -f "${SPACK_STACK_DIR}/configs/sites/tier2/${host}/packages_*.yaml" + fi + fi + spack env activate -p ${env_dir} + + if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then + echo "Running spack external find for macOS generic packages..." + spack external find --not-buildable autoconf automake bash cmake cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo + + brew_prefix=$(brew --prefix) + tcsh_version=$(${brew_prefix}/bin/tcsh --version | awk '{print $2}') + rust_version=$(${brew_prefix}/bin/rustc --version | awk '{print $2}') + + echo "Manually injecting tricky macOS packages into Spack configuration..." + cat << EOF > spack-macos-externals.yaml +packages: + tcsh: + externals: + - spec: tcsh@${tcsh_version} + prefix: ${brew_prefix} + rust: + externals: + - spec: rust@${rust_version} + prefix: ${brew_prefix} + extra_attributes: + cargo: ${brew_prefix}/bin/cargo + compilers: + rust: ${brew_prefix}/bin/rustc +EOF + spack config add -f spack-macos-externals.yaml + rm -f spack-macos-externals.yaml + fi + + echo "Registering bootstrap mirror ${bootstrap_mirror_path} ..." + if [[ ! -d ${bootstrap_mirror_path} ]]; then + echo "ERROR, directory ${bootstrap_mirror_path} not found" + exit 1 + fi + spack bootstrap list | grep local-sources || \ + spack bootstrap add --trust local-sources ${bootstrap_mirror_path}/metadata/sources + spack bootstrap list | grep local-binaries || \ + spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries + + # Check that the site has mirrors configured for local source and build caches, + # and extract the local path on disk. Need to strip leading "file://" from path + result=$(spack mirror list | grep local-source) || \ + (echo "ERROR, no local source cache configured" && exit 1) + source_mirror_path=$(echo ${result} | cut -d " " -f 3) + source_mirror_path=${source_mirror_path:7} + echo "Spack source mirror path: ${source_mirror_path}" + # For build caches, additional logic is needed. If buildcache_dir is defined, + # update the location of the default build cache to this directory. + result=$(spack mirror list | grep local-binary) || \ + (echo "ERROR, no local binary cache configured" && exit 1) + binary_mirror_path=$(echo ${result} | cut -d " " -f 3) + binary_mirror_path=${binary_mirror_path:7} + # If buildcache_dir is set, update binary_mirror_path + if [[ ! -z ${buildcache_dir} ]]; then + sed -i "s#${binary_mirror_path}#${buildcache_dir}#g" ${env_dir}/site/mirrors.yaml + result=$(spack mirror list | grep local-binary) + binary_mirror_path=$(echo ${result} | cut -d " " -f 3) + binary_mirror_path=${binary_mirror_path:7} + fi + echo "Spack binary mirror path: ${binary_mirror_path}" + + if [[ "${update_build_cache}" == "true" ]]; then + spack config add config:install_tree:padded_length:200 + fi + + # Bootstrap spack explicitly + echo "Bootstrapping spack ..." + spack bootstrap now 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.bootstrap.${env_name}.${LOG_TIMESTAMP} + + # Concretize environment, and check that spack.lock is created + spack concretize --force --fresh 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.concretize.${env_name}.${LOG_TIMESTAMP} + if [[ ! -e ${env_dir}/spack.lock ]]; then + echo "ERROR during concretization of environment ${env_name}, spack.lock not found" + exit 1 + fi + + # Check for duplicate packages + ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env + + # Update local source cache if requested + if [[ "${update_source_cache}" == "true"* ]]; then + echo "Updating local source cache ..." + spack mirror create -a -d ${source_mirror_path} + fi + + # Update local cargo mirror if requested; this can be + # unreliable, therefore ignore errors and proceed ... + if [[ "${update_cargo_mirror}" == "true"* ]]; then + set +e + echo "Updating local cargo mirror ..." + ./util/fetch_cargo_deps.py + set -e + fi + + # Install the environment with the correct flags + case ${reuse_build_cache} in + "true") + buildcache_install_flags="--no-check-signature" + ;; + "false") + buildcache_install_flags="--no-cache" + ;; + *) + echo "ERROR, unkown reuse_build_cache value ${reuse_build_cache} for setting install flags" + exit 1 + ;; + esac + + case ${submit_to_scheduler} in + "true") + jobs=$(tasks_per_node ${host}) + parallel_install_flags="--concurrent-packages=10 --jobs=${jobs}" + ;; + "false") + parallel_install_flags="" + ;; + *) + echo "ERROR, unkown submit_to_scheduler value ${submit_to_scheduler} for setting install flags" + exit 1 + ;; + esac + + install_script=${PWD}/spack-install.${env_name}.sh + + # Locally ignore the generated install script in git without changing global .gitignore + if [[ -d "${SPACK_STACK_DIR}/.git" ]] && ! grep -q "^spack-install\.\*\.sh$" "${SPACK_STACK_DIR}/.git/info/exclude" 2>/dev/null; then + echo "spack-install.*.sh" >> "${SPACK_STACK_DIR}/.git/info/exclude" + fi + + cat << EOF > ${install_script} +#!/usr/bin/env bash + +set -e + +$(declare -p test_packages) + +# If no tests are required, install everything +if [[ \${#test_packages[@]} -eq 0 ]]; then + set -o pipefail + spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP} + set +o pipefail +else + for (( idx=0; idx<\${#test_packages[@]}; idx++ )); do + test_package=\${test_packages[\${idx}]} + # First, check if this package is in this environment + set +e + grep -e "\${test_package}@" log.concretize.${env_name}.${LOG_TIMESTAMP} || continue + set -e + idx_padded=\$(printf "%03d" "\$((idx+1))") + set -o pipefail + spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} --only=dependencies \${test_package} \\ + 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded}.\${test_package}-dependencies + spack install --verbose --no-cache --test=root \${test_package} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded}.\${test_package} + set +o pipefail + done + # idx now equals the length of the array; install the rest + idx_padded=\$(printf "%03d" "\$((idx+1))") + set -o pipefail + spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded} + set +o pipefail +fi +EOF + chmod u+x ${install_script} + if [[ "${submit_to_scheduler}" == "true" ]]; then + run_interactive_job ${host} ${install_script} ${reuse_build_cache} + else + bash ${install_script} + fi + + # In build mode, update local binary cache + if [[ "${update_build_cache}" == "true" ]]; then + spack buildcache push -u ${binary_mirror_path} + spack buildcache update-index local-binary + fi + + # In install mode, create environment modules + if [[ "${update_build_cache}" == "false" ]]; then + spack module ${module_choice} refresh --yes --upstream-modules 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.modules.${env_name}.${LOG_TIMESTAMP} + spack stack setup-meta-modules 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.setup-meta-modules.${env_name}.${LOG_TIMESTAMP} + fi + + # When creating or updating buildcaches, fix permissions for mirrors. + # Mirrors do not contain executables, therefore skip looking for them. + if [[ "${update_source_cache}" == "true" ]]; then + fix_permissions ${host} ${source_mirror_path} 0 + fi + if [[ "${update_build_cache}" == "true" ]]; then + fix_permissions ${host} ${binary_mirror_path} 0 + fi + if [[ "${update_cargo_mirror}" == "true" ]]; then + fix_permissions ${host} ${cargo_mirror_path} 0 + fi + + # Clean up (don't remove software and configuration needed to bootstrap Spack) + spack clean -d -f -m -p -s + spack env deactivate + first_pass="false" + + done + +done + +# Repair permissions for environments if in installer mode +if [[ "${update_build_cache}" == "false" ]]; then + # Also search for exectuables + if [[ "${SPACK_STACK_DRY_RUN}" == "true" ]]; then + echo "[DRY-RUN] fix_permissions ${host} ${environment_dirs} 1" + else + fix_permissions ${host} ${environment_dirs} 1 + fi +fi + +echo "SUCCESS" +echo + +exit 0