.gitlab-ci.yml

image: alpine

variables:
  ## External images
  DOCKER_IMAGE: docker.io/docker:28.3.3
  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5

  ## CUDA version and container operating system
  CUDA_VERSION: 12.5.1
  CUDA_OS: ubuntu24.04

  ## Default versions are specified in packages.yaml but can be overridden
  EDM4EIC_VERSION: ""
  EICRECON_VERSION: ""
  EPIC_VERSION: ""
  JUGGLER_VERSION: ""

  ## Local registry
  CI_PUSH: 1
  ## Dockerhub registry
  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 1
  ## GitHub registry
  GH_REGISTRY: ghcr.io
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

  ## Number of jobs to start during container builds
  JOBS: 32

  ## is this nightly or not?
  NIGHTLY: ""
  ## Add to tag
  NIGHTLY_TAG: "nightly"

  ## Additional options for docker build (e.g. --no-cache)
  BUILD_OPTIONS: ""

  ## Version to be set by external trigger
  VERSION: ""

  ## Internal tag used for the CI
  INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"

  PIPELINE_NAME: "$CI_COMMIT_TITLE"

  ## Variables passed from GitHub workflows
  GITHUB_REPOSITORY: ""
  GITHUB_SHA: ""
  GITHUB_PR: ""

stages:
  - status-pending
  - config
  - base            ## base OS image
  - eic             ## EIC container images
  - deploy          ## build/deploy singularity images
  - benchmarks
  - test
  - finalize
  - status-report

## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## nightly builds are now part of the regular master build in order to keep
## all artifacts available at all times.
workflow:
  name: '$PIPELINE_NAME'
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_PIPELINE_SOURCE == "webide"'
    - if: '$CI_PIPELINE_SOURCE == "trigger"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-(alpha|beta|stable)/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-(alpha|beta|stable)/'   ## special stable branch: vX.Y-acadia-stable (etc)
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'            ## commit tags start with vX.Y.Z with optional suffix

## plan:
##  Workflows:
##  - master      --> config + all build stages + singularity
#                            + nightly eic + nightly singularity
##  - v4.0-stable --> config + all build stages + singularity
##  - v4.0.0      --> config + all build stages + singularity
##  - MR          --> config + all build stages
##
##  Container images tags
##  - master      --> testing
##  - <nightly>   --> nightly (run as part of master)
##  - v4.0-stable --> 4.0-stable
##  - v4.0.0      --> 4.0-stable, 4.0.0
##  - acadia
##  - MR          --> 4.0-unstable (on all registries)
##                --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
##  - all other   --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e

nvidia-smi:
  stage: config
  image: nvidia/cuda:${CUDA_VERSION}-base-${CUDA_OS}
  tags:
    - gpu
  variables:
    NVIDIA_DISABLE_REQUIRE: "true"
  script:
    - |
      nvidia-smi
  allow_failure: true

## Default version and tags
## - internal tag used for the CI. Also temporarily tagged
##   on eicweb to communicate between jobs (removed in cleanup job)
## - export tag to public registries, optional secondary export tag
version:
  stage: config
  script:
    - |
      if [ -n "${VERSION}" ]; then
        # external trigger with specified version
        EXPORT_TAG="${VERSION}"
      elif [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
        NIGHTLY_TAG="${VERSION}-nightly"
        EXPORT_TAG="${VERSION}"
        DH_PUSH=""
      elif [ "x${CI_PIPELINE_SOURCE}" = "xtrigger" ]; then
        if [[ "${GITHUB_REPOSITORY}" == "eic/containers" ]] ; then
          VERSION="unstable-mr-${GITHUB_PR}"
          NIGHTLY_TAG="${VERSION}-nightly"
          EXPORT_TAG="${VERSION}"
        else
          VERSION="${CI_COMMIT_BRANCH}"
          GH_PUSH=""
        fi
        DH_PUSH=""
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
        EXPORT_TAG="${VERSION}"
        GEOM_VERSION="$(echo "$VERSION" | sed -e 's/-.\+$//')" # cut out suffixes like -stable
        for ENV_FILE in spack-environment/xl/epic/spack.yaml spack-environment/cuda/epic/spack.yaml; do
          grep -- "- epic@${GEOM_VERSION}" "${ENV_FILE}" >/dev/null \
            || ( echo "Unable to locate epic release package for ${GEOM_VERSION} in ${ENV_FILE}"; exit 1; )
        done
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-\(alpha\|beta\|stable\) ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-\(alpha\|beta\|stable\) ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
        VERSION="${CI_COMMIT_BRANCH}"
        EXPORT_TAG="${VERSION}"
      else
        VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}"
        EXPORT_TAG=""
      fi
      echo "INTERNAL_TAG=$INTERNAL_TAG" > build.env
      echo "VERSION=$VERSION" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
      echo "DH_PUSH=$DH_PUSH" >> build.env
      echo "GH_PUSH=$GH_PUSH" >> build.env
      cat build.env
  artifacts:
    reports:
      dotenv: build.env

.status:
  image: curlimages/curl:latest
  rules:
    - if: $GITHUB_SHA !~ /^$/ # when GITHUB_SHA non-empty
  before_script: []
  script:
    - |
      curl \
        -X POST \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: token ${GITHUB_REPO_STATUS_TOKEN}" \
        "https://api.github.com/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \
        -d '{"state":"'"${STATE}"'",
              "target_url":"'"${CI_PIPELINE_URL}"'",
              "description":"'"${DESCRIPTION} $(TZ=America/New_York date)"'",
              "context":"eicweb/eic_container"
            }' ;

status:pending:
  stage: status-pending
  extends: .status
  variables:
    STATE: "pending"
    DESCRIPTION: "Started..."
  when: always

## base job settings for all docker interactions
.docker:
  image: ${DOCKER_IMAGE}
  variables:
    DOCKER_HOST: unix:///var/run/docker.sock
    DOCKER_TLS_CERTDIR: ""
  before_script:
    - # Login to local registry
      echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
    - # Login to remote registries
      if [[ -n "${EXPORT_TAG}" ]] ; then
        if [[ -n "${DH_REGISTRY}" ]] ; then
          echo ${DH_EICWEB_TOKEN} | docker login -u ${DH_REGISTRY_USER} --password-stdin ${DH_REGISTRY} ;
        fi ;
        if [[ -n "${GH_REGISTRY}" ]] ; then
          echo ${GITHUB_REGISTRY_TOKEN} | docker login -u ${GITHUB_REGISTRY_USER} --password-stdin ${GH_REGISTRY} ;
        fi ;
      fi
  after_script:
    - # Logout from local registry
      docker logout ${CI_REGISTRY}
    - # Logout from remote registries
      if [[ -n "${EXPORT_TAG}" ]] ; then
        if [[ -n "${DH_REGISTRY}" ]] ; then
          docker logout ${DH_REGISTRY} ;
        fi ;
        if [[ -n "${GH_REGISTRY}" ]] ; then
          docker logout ${GH_REGISTRY} ;
        fi ;
      fi

## base job settings for all docker build jobs
.build:
  extends: .docker
  rules:
    - when: on_success
  tags:
    - docker-new
  before_script:
    - !reference [.docker, before_script]
    - if [ "$SKIP_BINFMT" != "true" ]; then
        mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
        for arch in aarch64 ; do
          if ! grep -q enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
            docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes ;
          fi ;
        done ;
      fi
    - docker context create context
    # The docker-container cache persistence applies to builders of the same name,
    # so do not change the name to something that has the pipeline or job id.
    - docker buildx create context --name builder --driver docker-container --bootstrap --use
    - docker buildx ls
    - echo $DOCKER_HOST
  after_script:
    - !reference [.docker, after_script]
    - docker buildx ls
  artifacts:
    paths:
      - build.log
    when: always

## rules for nightly jobs
.nightly:
  rules:
    # Don't build nightly in the following cases
    - if: '$BUILD_TYPE == "nightly" && $CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-(alpha|beta|stable)/'          ## main stable branch: vX.Y-stable
      when: never
    - if: '$BUILD_TYPE == "nightly" && $CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-(alpha|beta|stable)/'   ## special stable branch: vX.Y-acadia-stable (etc)
      when: never
    - if: '$BUILD_TYPE == "nightly" && $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'                         ## commit tags start with vX.Y.Z with optional suffix
      when: never
    # Skip containers other than eic_ci for epic or EICrecon trigger
    - if: '$ENV != "ci" && $CI_PIPELINE_SOURCE == "trigger" && $GITHUB_REPOSITORY != "eic/containers"'
      when: never
    # Caller needs to provide their own whitelist
    #- when: always

## Images:
## debian_testing_base --> eic_dev  --> eic_xl
##             ---------------> eic_sim
##             ---------------> eic_ml
## debian_stable_base --> oneapi_eic_dev
## oneapi_jug_dev + eic_xl --> oneapi_eic_xl

## TODO
## oneapi_runtime + eic_xl --> oneapi_prod

base:
  parallel:
    matrix:
#      - BASE_IMAGE: debian:testing-slim
#        BUILD_IMAGE: debian_testing_base
#        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: debian:trixie-slim
        BUILD_IMAGE: debian_stable_base
        PLATFORM: linux/amd64
#      - BASE_IMAGE: ubuntu:22.04
#        BUILD_IMAGE: ubuntu_base
#        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
#        BUILD_IMAGE: oneapi_base
#        PLATFORM: linux/amd64
      - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-devel-${CUDA_OS}
        BUILD_IMAGE: cuda_devel
        PLATFORM: linux/amd64
      - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS}
        BUILD_IMAGE: cuda_runtime
        PLATFORM: linux/amd64
  extends: .build
  stage: base
  needs:
    - version
  script:
    - source spack.sh ;
      source spack-packages.sh ;
      source key4hep-spack.sh ;
      source eic-spack.sh ;
    - docker buildx build --push ${BUILD_OPTIONS}
                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/debian/Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
                   --build-arg NVIDIA_VISIBLE_DEVICES=$(case "${BUILD_IMAGE}" in cuda*) echo all ;; esac)
                   --build-arg NVIDIA_DRIVER_CAPABILITIES=$(case "${BUILD_IMAGE}" in cuda*) echo all ;; esac)
                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
                   --build-arg SPACK_VERSION=${SPACK_VERSION}
                   --build-arg SPACK_SHA=$(sh .ci/resolve_git_ref "${SPACK_ORGREPO}" "${SPACK_VERSION}")
                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
                   --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
                   --build-arg SPACKPACKAGES_ORGREPO=${SPACKPACKAGES_ORGREPO}
                   --build-arg SPACKPACKAGES_VERSION=${SPACKPACKAGES_VERSION}
                   --build-arg SPACKPACKAGES_SHA=$(sh .ci/resolve_git_ref "${SPACKPACKAGES_ORGREPO}" "${SPACKPACKAGES_VERSION}")
                   --build-arg SPACKPACKAGES_CHERRYPICKS="${SPACKPACKAGES_CHERRYPICKS}"
                   --build-arg SPACKPACKAGES_CHERRYPICKS_FILES="${SPACKPACKAGES_CHERRYPICKS_FILES}"
                   --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
                   --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
                   --build-arg KEY4HEPSPACK_SHA=$(sh .ci/resolve_git_ref "${KEY4HEPSPACK_ORGREPO}" "${KEY4HEPSPACK_VERSION}")
                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
                   --build-arg EICSPACK_SHA=$(sh .ci/resolve_git_ref "${EICSPACK_ORGREPO}" "${EICSPACK_VERSION}")
                   --build-arg jobs=${JOBS}
                   --build-arg BUILDWEEK=$(date +%Yw%V)
                   --provenance false
                   containers/debian
                   2>&1 | tee build.log

eic:
  parallel:
    matrix:
      - BUILD_IMAGE: eic_
        ENV:
        - ci
        - cvmfs
        - dbg
        - jl
        - prod
        - xl
        BUILD_TYPE:
        - default
        BUILDER_IMAGE: debian_stable_base
        RUNTIME_IMAGE: debian_stable_base
        PLATFORM: linux/amd64
      - BUILD_IMAGE: eic_
        ENV:
        - ci
        - ci_without_acts
        - cvmfs
        - xl
        BUILD_TYPE:
        - nightly
        BUILDER_IMAGE: debian_stable_base
        RUNTIME_IMAGE: debian_stable_base
        PLATFORM: linux/amd64
#      - BUILD_IMAGE: oneapi_
#        ENV:
#        - prod
#        BUILD_TYPE:
#        - default
#        - nightly
#        BUILDER_IMAGE: oneapi_base
#        RUNTIME_IMAGE: oneapi_base
#        PLATFORM: linux/amd64
      - BUILD_IMAGE: eic_
        ENV:
        - cuda
        - tf
        BUILD_TYPE:
        - default
        - nightly
        BUILDER_IMAGE: cuda_devel
        RUNTIME_IMAGE: cuda_runtime
        PLATFORM: linux/amd64
      - BUILD_IMAGE: eic_dev_
        ENV:
        - cuda
        BUILD_TYPE:
        - default
        - nightly
        BUILDER_IMAGE: cuda_devel
        RUNTIME_IMAGE: cuda_devel
        PLATFORM: linux/amd64
  rules:
    - !reference ['.nightly', rules]
    - when: always
  extends: .build
  stage: eic
  needs:
    - version
    - base
  script:
    - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi
    - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi
    - apk add envsubst git
    - source spack.sh ;
      source spack-packages.sh ;
      source key4hep-spack.sh ;
      source eic-spack.sh ;
      export SPACKPACKAGES_VERSION ;
      cat mirrors.yaml.in | envsubst > mirrors.yaml
    - set -o xtrace ;
      docker buildx build --push ${BUILD_OPTIONS}
                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_COMMIT_REF_SLUG}-amd64
                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_COMMIT_REF_SLUG}-amd64
                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
                   ${EXPORT_TAG:+
                     ${IF_BUILD_DEFAULT+
                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
                     }
                     ${IF_BUILD_DEFAULT-
                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                     }
                   }
                   ${IF_BUILD_NIGHTLY+
                     ${NIGHTLY:+
                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                     }
                   }
                   --file containers/eic/Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BENCHMARK_COM_SHA=$(sh .ci/resolve_git_ref https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git master)
                   --build-arg BENCHMARK_DET_SHA=$(sh .ci/resolve_git_ref https://eicweb.phy.anl.gov/EIC/benchmarks/detector_benchmarks.git master)
                   --build-arg BENCHMARK_REC_SHA=$(sh .ci/resolve_git_ref https://eicweb.phy.anl.gov/EIC/benchmarks/reconstruction_benchmarks.git master)
                   --build-arg BENCHMARK_PHY_SHA=$(sh .ci/resolve_git_ref https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks.git master)
                   --build-arg CAMPAIGNS_HEPMC3_SHA=$(sh .ci/resolve_git_ref eic/simulation_campaign_hepmc3 main)
                   --build-arg CAMPAIGNS_CONDOR_SHA=$(sh .ci/resolve_git_ref eic/job_submission_condor main)
                   --build-arg CAMPAIGNS_SLURM_SHA=$(sh .ci/resolve_git_ref eic/job_submission_slurm main)
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BUILDER_IMAGE=${BUILDER_IMAGE}
                   --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg EIC_CONTAINER_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
                   --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
                   ${IF_BUILD_DEFAULT+
                     ${EDM4EIC_VERSION:+--build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION})}
                     ${EICRECON_VERSION:+--build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION})}
                     ${EPIC_VERSION:+--build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION})}
                     ${JUGGLER_VERSION:+--build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION})}
                   }
                   ${IF_BUILD_NIGHTLY+
                     --build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main})
                     --build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main})
                     --build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main})
                     --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main})
                   }
                   --build-arg ENV=${ENV}
                   --build-arg SPACK_DUPLICATE_ALLOWLIST=$(
                     case "${ENV}" in
                       ci|ci_without_acts|cuda|dbg|jl|prod)
                         echo "epic|llvm|py-setuptools|py-urllib3" ;;
                       xl|tf)
                         echo "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" ;;
                       *)
                         echo "epic|llvm|py-setuptools|py-urllib3" ;;
                     esac
                   )
                   --build-arg jobs=${JOBS}
                   --build-context spack-environment=spack-environment
                   --secret id=mirrors,src=mirrors.yaml
                   --secret type=env,id=CI_REGISTRY_USER,env=CI_REGISTRY_USER
                   --secret type=env,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD
                   --secret type=env,id=GITHUB_REGISTRY_USER,env=GITHUB_REGISTRY_USER
                   --secret type=env,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN
                   --provenance false
                   containers/eic
                   2>&1 | tee build.log

user_spack_environment:
  stage: benchmarks
  rules:
    # Skip for epic or EICrecon trigger
    - if: '$CI_PIPELINE_SOURCE == "trigger" && $GITHUB_REPOSITORY != "eic/containers"'
      when: never
    - when: always
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_
            ENV: ci
            BUILD_TYPE: default
            BUILDER_IMAGE: debian_stable_base
            RUNTIME_IMAGE: debian_stable_base
            PLATFORM: linux/amd64
  image:
    name: ${CI_REGISTRY_IMAGE}/eic_ci:${INTERNAL_TAG}-default
  script:
    - source /opt/spack/share/spack/setup-env.sh
    - spack env activate --create edm4eic
    - spack add edm4eic@git.main cxxstd=20
    - spack concretize
    - spack install --only dependencies edm4eic
    - ls -al /opt/software/linux-*/compiler-wrapper-1.0-*/libexec/spack/gcc/g++
    - spack install --verbose
    - ls $HOME/spack/

.cuda:
  stage: benchmarks
  tags:
    - gpu
  rules:
    # Only eic_ci container is built for external trigger
    - if: '$CI_PIPELINE_SOURCE == "trigger"'
      when: never
    - when: always
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_dev_
            ENV: cuda
            BUILD_TYPE: default
            BUILDER_IMAGE: cuda_devel
            RUNTIME_IMAGE: cuda_devel
            PLATFORM: linux/amd64
  image:
    name: ${CI_REGISTRY_IMAGE}/eic_dev_cuda:${INTERNAL_TAG}-default

cuda:torch:
  extends: .cuda
  script:
    - python -c "import torch ; print(torch.cuda.is_available()) ;"
  allow_failure: true

.singularity:
  image:
    name: ${SINGULARITY_IMAGE}
    entrypoint: [""]
  stage: deploy
  interruptible: true
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE != "trigger"'
      when: on_success
    - when: never
  artifacts:
      expire_in: 1 days
      paths:
        - build/${BUILD_IMAGE}.sif
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE}

eic_xl:singularity:default:
  extends: .singularity
  variables:
    BUILD_TYPE: default
    BUILD_IMAGE: eic_xl
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_
            ENV: xl
            BUILD_TYPE: default
            BUILDER_IMAGE: debian_stable_base
            RUNTIME_IMAGE: debian_stable_base
            PLATFORM: linux/amd64

# eic-shell/install.py expects build/eic_xl.sif from the job with name 'eic_xl:singularity:nightly'
eic_xl:singularity:nightly:
  extends: .singularity
  variables:
    BUILD_TYPE: nightly
    BUILD_IMAGE: eic_xl
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_
            ENV: xl
            BUILD_TYPE: nightly
            BUILDER_IMAGE: debian_stable_base
            RUNTIME_IMAGE: debian_stable_base
            PLATFORM: linux/amd64

.benchmarks:
  stage: benchmarks
  variables:
    DETECTOR: "epic"
    DETECTOR_CONFIG: epic_craterlake
    DETECTOR_REPOSITORYURL: 'https://github.com/eic/epic.git'
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
    PIPELINE_NAME: "eic_container: $CI_PIPELINE_NAME"
    GITHUB_REPOSITORY: "$GITHUB_REPOSITORY"
    GITHUB_SHA: "$GITHUB_SHA"
    GITHUB_PR: "$GITHUB_PR"
  allow_failure: false

.benchmarks:default:
  extends: .benchmarks
  variables:
    BUILD_TYPE: default
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_
            ENV: ci
            BUILD_TYPE: default
            BUILDER_IMAGE: debian_stable_base
            RUNTIME_IMAGE: debian_stable_base
            PLATFORM: linux/amd64

.benchmarks:nightly:
  extends: .benchmarks
  variables:
    BUILD_TYPE: nightly
  needs:
    - job: version
    - job: eic
      parallel:
        matrix:
          - BUILD_IMAGE: eic_
            ENV: ci
            BUILD_TYPE: nightly
            BUILDER_IMAGE: debian_stable_base
            RUNTIME_IMAGE: debian_stable_base
            PLATFORM: linux/amd64

benchmarks:geoviewer:default:
  extends: .benchmarks:default
  rules:
   - if: '$CI_PIPELINE_SOURCE != "schedule" && $EPIC_VERSION != ""'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
    project: EIC/benchmarks/geoviewer
    strategy: depend

benchmarks:detector:default:
  extends: .benchmarks:default
  rules:
   - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
    project: EIC/benchmarks/detector_benchmarks
    strategy: depend

benchmarks:detector:nightly:
  extends: .benchmarks:nightly
  rules:
    - !reference ['.nightly', rules]
    - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
  trigger:
    project: EIC/benchmarks/detector_benchmarks
    strategy: depend

benchmarks:phyiscs:default:
  extends: .benchmarks:default
  rules:
   - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend

benchmarks:physics:nightly:
  extends: .benchmarks:nightly
  rules:
    - !reference ['.nightly', rules]
    - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend

benchmarks:reconstruction:default:
  extends: .benchmarks:default
  rules:
   - if: '$CI_PIPELINE_SOURCE != "schedule" && $EICRECON_VERSION == ""'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend

benchmarks:reconstruction:nightly:
  extends: .benchmarks:nightly
  rules:
    - !reference ['.nightly', rules]
    - if: '$CI_PIPELINE_SOURCE != "schedule" && $EICRECON_VERSION == ""'
  variables:
    BENCHMARKS_CONTAINER: eic_ci
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend

df:
  extends: .build
  stage: config
  tags:
    - docker-new
  needs:
    - version
  rules:
    - when: manual
  script:
    - df -h
    - docker images
    - docker ps --size
    - docker stats --all --no-stream --no-trunc > docker-stats-all.txt
    - docker context ls
    - docker volume ls
    - docker volume ls --filter=name=buildx_buildkit_builder
    - docker buildx ls
    - docker buildx inspect
    - docker buildx du
    - docker buildx du --verbose > docker-buildx-du-verbose.txt
    - docker system df
    - docker system df --verbose > docker-system-df-verbose.txt
  artifacts:
    paths:
      - docker-stats-all.txt
      - docker-buildx-du-verbose.txt
      - docker-system-df-verbose.txt
  allow_failure: true

.prune:
  extends: .build
  stage: config
  rules:
    - when: manual
  script:
    - docker system prune --all --volumes --force
    - docker buildx prune --all --force --filter=until=24h --verbose
    - docker system prune --all --force --filter=until=24h
  allow_failure: true

prune:gpu:
  extends: .prune
  tags:
    - gpu
  variables:
    NVIDIA_DISABLE_REQUIRE: "true"
    SKIP_BINFMT: "true"

prune:docker-new:
  extends: .prune
  tags:
    - docker-new

clean_internal_tag:
  image: alpine/curl
  stage: finalize
  rules:
    - when: manual
  script:
    - |
      for id in 66 68 69 80 83 91 92 97 ; do
        for tag in ${INTERNAL_TAG} ${INTERNAL_TAG}-default ${INTERNAL_TAG}-nightly ; do
          curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${id}/tags/${tag} ;
        done ;
      done
  allow_failure: true

.clean_unstable_mr:
  extends: .docker
  stage: finalize
  when: always
  script:
    - apk add curl jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do
        docker images
          --filter=reference=${registry}/*:unstable-mr*
          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
      done
      | sed -n -E 's/unstable-mr-([0-9]+)-([0-9]+)(-nightly|-default){0,1}$/& \1 \2/p'
      | while read id repository tag project_id merge_request_iid ; do
        state=$(curl --silent -L
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=opened&per_page=100"
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
                | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") ;
        echo "Image $repository:$tag is ${state:-undetermined} (project ${project_id}, iid ${merge_request_iid})." ;
        if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
      done
  allow_failure: true

clean_unstable_mr:gpu:
  extends: .clean_unstable_mr
  tags:
    - gpu
  variables:
    NVIDIA_DISABLE_REQUIRE: "true"
    SKIP_BINFMT: "true"

clean_unstable_mr:docker-new:
  extends: .clean_unstable_mr
  tags:
    - docker-new

.clean_pipeline:
  extends: .docker
  stage: finalize
  needs:
    - version
    - clean_internal_tag
  when: always
  script:
    - apk add curl jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do
        docker images
          --filter=reference=${registry}/*:pipeline-*
          --format "{{.ID}} {{.Repository}} {{.Tag}} {{.CreatedSince}}" ;
      done
    - # prevent grep exit code 1 when no match
      mygrep() { grep "$@" || test $? = 1; } ;
      for registry in
        eicweb
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do
        docker images
          --filter=reference=${registry}/*:pipeline-*
          --format "{{.ID}} {{.Repository}} {{.Tag}} {{.CreatedSince}}" ;
      done
      | mygrep -v ${CI_PIPELINE_ID}
      | mygrep -v "hours ago"
      | sed -n -E 's/pipeline-([0-9]+)(-.*){0,1}\s+.*$/\1 &/p'
      | while read id repository pipeline_id tag N durations ago ; do
        echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${pipeline_id}" ;
        curl --silent -L "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${pipeline_id}" | jq . ;
        status=$(curl --silent -L
                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${pipeline_id}"
                | jq -r ".status") ;
        echo "Image $repository:$tag is ${status:-undetermined} (pipeline ${pipeline_id})." ;
        if [ "$status" == "success" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$status" == "failed" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$status" == "canceled" ] ; then docker rmi $repository:$tag ; fi ;
      done
  allow_failure: true

clean_pipeline:gpu:
  extends: .clean_pipeline
  tags:
    - gpu
  variables:
    NVIDIA_DISABLE_REQUIRE: "true"
    SKIP_BINFMT: "true"

clean_pipeline:docker-new:
  extends: .clean_pipeline
  tags:
    - docker-new

status:success:
  stage: status-report
  dependencies: []
  extends: .status
  variables:
    STATE: "success"
    DESCRIPTION: "Succeeded!"
  when: on_success

status:failure:
  stage: status-report
  dependencies: []
  extends: .status
  variables:
    STATE: "failure"
    DESCRIPTION: "Failed!"
  when: on_failure