From 08f8854ec9a0e55593b3004622d729adadd6dd11 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 21 Apr 2024 21:14:07 +0200 Subject: [PATCH 1/7] enable mounting previous overlay-upper read-only --- eessi_container.sh | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 7d00d1400c..8430a44d2c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -594,11 +594,47 @@ declare -a EESSI_FUSE_MOUNTS=() # always mount cvmfs-config repo (to get access to software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +# check if we got some data via --resume and, if so, use the overlayfs +# example scenario: +# 1st step: some software is build in rw mode +# 2nd step: the software is tested in ro mode (same access as when we would use a +# repository) if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + if [[ -d ${EESSI_TMPDIR}/overlay-upper ]]; then + # the overlay-upper directory is only created in a read-write-session, thus + # we are resuming from such a session here (otherwise there shouldn't be such + # directory yet as it is only created for read-write-sessions a bit further + # below); the overlay-upper directory can only exist because it is part of + # the ${RESUME} directory or tarball + # to be able to see the contents of the read-write session we have to mount + # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + + # make sure the overlay-upper directory exists + mkdir -p ${EESSI_TMPDIR}/overlay-upper + + # make the target CernVM-FS repository available under /cvmfs_ro + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + + # now, put the overlay-upper read-only on top of the repo and make it under the usual prefix /cvmfs available + EESSI_READONLY_OVERLAY="container:fuse-overlayfs" + # ${EESSI_TMPDIR} is bind mounted to /tmp, hence ${EESSI_TMPDIR}/overlay-upper becomes available as /tmp/overlay-upper + # the left-most lower dir is put on top, with no upperdir=... the whole overlayfs is made available read-only + EESSI_READONLY_OVERLAY+=" -o lowerdir=/tmp/overlay-upper:/cvmfs_ro/${repo_name}" + EESSI_READONLY_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_READONLY_OVERLAY + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + # no overlay-upper directory means we are in a plain read-only session and + # don't need any fuse-overlayfs + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + fi fi if [[ "${ACCESS}" == "rw" ]]; then From d043d867a3dccce1b978ee2531923e970b6e6176 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 12:10:19 +0200 Subject: [PATCH 2/7] Add `psm2` to filtered dependencies for `2023.06` `PSM2` was introduced as a dependency of `libfabric` in https://github.com/easybuilders/easybuild-easyconfigs/pull/20501. We already have PSM2 in the compat layer, so we can filter this dependency out, but longer term we probably actually want it since it should be built with accelerator support. --- configure_easybuild | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index c67b879cf3..60e3744603 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -32,6 +32,12 @@ if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then DEPS_TO_FILTER="${DEPS_TO_FILTER},Yasm" fi +# Version 23.06 of EESSI ships PSM2 in the compat layer, so we can filter this out while retaining support for OFA fabric +# (longer term this is probably not the right move as PSM2 should be configured with accelerator support, hence the restricted version) +if [[ "$EESSI_VERSION" == "2023.06" ]]; then + DEPS_TO_FILTER="${DEPS_TO_FILTER},psm2" +fi + export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 From ec2e8bdf04456b85bf9dae3a46ea132acba90df3 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 12:14:20 +0200 Subject: [PATCH 3/7] Update configure_easybuild --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 60e3744603..ed3e651a4c 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -35,7 +35,7 @@ fi # Version 23.06 of EESSI ships PSM2 in the compat layer, so we can filter this out while retaining support for OFA fabric # (longer term this is probably not the right move as PSM2 should be configured with accelerator support, hence the restricted version) if [[ "$EESSI_VERSION" == "2023.06" ]]; then - DEPS_TO_FILTER="${DEPS_TO_FILTER},psm2" + DEPS_TO_FILTER="${DEPS_TO_FILTER},PSM2" fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER From 3cdb7b2653336dd707b65197bf578f4012a6b0ff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 12:43:43 +0200 Subject: [PATCH 4/7] Add a rebuild of GROMACS --- ...240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml new file mode 100644 index 0000000000..e767c63f6e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -0,0 +1,11 @@ +# 2024.05.15 +# Originally shipped version forgot to bump the gmxapi version and source +# tarball, it was still using an older version from the 2023.3 tarball. Looking +# at https://gitlab.com/gromacs/gromacs/-/blob/v2024.1/python_packaging/gmxapi/src/gmxapi/version.py?ref_type=tags#L68, +# the 2024.1 release includes gmxapi 0.5.0. +# +# See https://github.com/easybuilders/easybuild-easyconfigs/pull/20522 +easyconfigs: + - GROMACS-2024.1-foss-2023b.eb: + options: + from-pr: 20522 From fb329a4bfe535cffc8d70958fc65fa0072180416 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 12:57:36 +0200 Subject: [PATCH 5/7] Also add new dependency on scikit-build-core --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index e767c63f6e..4eca76fd66 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -4,8 +4,13 @@ # at https://gitlab.com/gromacs/gromacs/-/blob/v2024.1/python_packaging/gmxapi/src/gmxapi/version.py?ref_type=tags#L68, # the 2024.1 release includes gmxapi 0.5.0. # +# This also introduced a new build dependency on scikit-build-core for GROMACS +# # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20522 easyconfigs: + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: + options: + from-pr: 20526 - GROMACS-2024.1-foss-2023b.eb: options: from-pr: 20522 From 792ea3ebdbf011665f3b673ecd974b93e961f8c1 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 15:01:57 +0200 Subject: [PATCH 6/7] Use from-commit for merged PRs --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index 4eca76fd66..93d87bbcf3 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -10,7 +10,7 @@ easyconfigs: - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: options: - from-pr: 20526 + from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 - GROMACS-2024.1-foss-2023b.eb: options: - from-pr: 20522 + from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 From 29dd14d8c87bb2b27b453620ef4188400a7a0b5f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 15 May 2024 15:48:35 +0200 Subject: [PATCH 7/7] use from-pr instead of from-commit --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index 93d87bbcf3..eacfad7079 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -10,7 +10,9 @@ easyconfigs: - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: options: - from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 + # from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 + from-pr: 20526 - GROMACS-2024.1-foss-2023b.eb: options: - from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + # from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + from-pr: 20522