.gitlab-ci.yml

# yaml-language-server: $schema=gitlab-ci
# yaml-language-server: $format.enable=false

stages:
  - environment
  - compilers
  - concretize
  - install
  - deploy

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_TAG =~ /^v[0-9].*$/

variables:
  TERM: ansi

.parallel_definition:
  parallel:
    matrix:
      - environment: [helvetios]
        slurm_options: ["-c 36"]
        stack: ["pinot-noir"]
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']

      - environment: [izar]
        slurm_options: ['-c 40 --gpus 2 -p izar']
        apptainer_options: ['--nv']
        stack: ["pinot-noir"]
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['izar/']

      - environment: [jed]
        slurm_options: ["-c 72 -p jed -q jed"]
        stack: ["pinot-noir"]
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']

      - environment: [kuma-h100]
        slurm_options: ["-c 64"]
        stack: ["pinot-noir"]
        apptainer_options: ['--nv']
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']

      - environment: [kuma-l40s]
        slurm_options: ["-c 64"]
        stack: ["pinot-noir"]
        apptainer_options: ['--nv']
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']
  tags:
    - ${environment}
    - stack
  variables:
    environment: ${environment}
    CI_DATA_ST: "/scratch/${path_suffix}scitas-stack/ci"
    CI_DATA_LT: "/work/scitas-ge/scitas-stack/ci"
    COMMAND_OPTIONS_SBATCH: ${slurm_options}
    PYTHONUNBUFFERED: 1
  image: ${app_image}
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
.squashfs_var:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      variables:
        SQUASHFS_ID: ${CI_MERGE_REQUEST_IID}
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      variables:
        SQUASHFS_ID: ${CI_COMMIT_BRANCH}
    - if: $CI_COMMIT_TAG =~ /^v[0-9].*$/
      variables:
        SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}

.parallel_job:
  extends:
    - .parallel_definition
    - .squashfs_var
  variables:
    FAKEHOME: "${CI_DATA_ST}/homes"
    MOUNT_POINT: $(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
    STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
    image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
    squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
    APPTAINER_EXEC_OPTIONS: >-
      ${apptainer_options}
         --cleanenv
         -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un)
         --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache
         --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
         --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache
         --bind ${CI_DATA_ST}/overlayfs:/overlayfs
         --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}"
         --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
spack:checkout:
  stage: .pre
  variables:
    APPTAINER_EXEC_OPTIONS: >-
      --cleanenv
      --bind ${CI_DATA_LT}
      --bind ${CI_DATA_ST}
      --bind ${CI_DATA_LT}
  extends:
    - .parallel_definition
  before_script:
    - git config --global --add --bool advice.detachedHead false
  script:
    - ./ci/prepare_squashfs.sh
  timeout: 1h

spack:setup:
  stage: environment
  extends:
    - .parallel_job
  script:
    - ci/setup_spack.sh
    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  needs:
    - job: spack:checkout

spack:install_compilers:
  stage: compilers
  extends:
    - .parallel_job
  script:
    - ci/install_compilers.sh
    - source ci/stack_env.sh
    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} config blame compilers
    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  artifacts:
    reports:
      junit: spack-install-*.xml
  needs:
    - job: spack:setup
  timeout: 5h

spack:concretize:
  stage: concretize
  extends:
    - .parallel_job
  script:
    - source ci/stack_env.sh

    - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
          config blame | tee config-${environment}-${stack}.log

    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
          concretize | tee concretize-${environment}-${stack}.log

    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
          mirror create -D -d ${MOUNT_POINT}/spack-mirror -a || /usr//bin/true

    - cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock
    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  artifacts:
    paths:
      - config-*.log
      - concretize-*.log
      - spack-*.lock
  needs:
    - job: spack:install_compilers
  timeout: 24h

spack:install:
  stage: install
  extends:
    - .parallel_job
  script:
    - source ci/stack_env.sh

    # Cleaning patches in case one has a different hash
    - ${STACK_LOCATION}/spack/bin/spack
          --color always
          -e ${environment}
          clean -m

    - ${STACK_LOCATION}/spack/bin/spack
          --color always
          -e ${environment} install
          --log-file ${CI_PROJECT_DIR}/spack-install-${environment}.xml
          --log-format junit
          --only-concrete
          --fail-fast
          --show-log-on-error

    - ${STACK_LOCATION}/spack/bin/spack
          --color always
          -e ${environment}
          find -vl | tee spack-find-${environment}-${stack}.log

    - ${STACK_LOCATION}/spack/bin/spack
          --color always
          -e ${environment}
          buildcache create
          --update-index
          --key ${GPG_KEY_ID}
          ${MOUNT_POINT}/buildcache

    - ${STACK_LOCATION}/spack/bin/spack
          -e ${environment} module
          lmod refresh --yes | tee spack-modules-${environment}-${stack}.log

    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  needs:
    - job: spack:concretize
  artifacts:
    paths:
      - spack-*.log
      - spack-*.xml
    reports:
      junit: spack-install-*.xml
  timeout: 72h

spack:mksquashfs:
  stage: deploy
  extends:
    - .parallel_job
  script:
    - source ci/stack_env.sh
    - mkfir -p /squashfs-cache/releases/
    - mksquashfs ${STACK_LOCATION} /squashfs-cache/releases/${stack}-${environment}-${SQUASHFS_ID}-${CI_COMMIT_TAG}.sqfs
  needs:
    - job: spack:install
  timeout: 1h
  rules:
    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /v[0-9]+.*/