-
Nicolas Richart authoredNicolas Richart authored
.gitlab-ci.yml 6.94 KiB
# yaml-language-server: $schema=gitlab-ci
# yaml-language-server: $format.enable=false
stages:
- environment
- compilers
- concretize
- install
- deploy
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_TAG =~ /^v[0-9].*$/
variables:
TERM: ansi
CI_DATA_LT: "/work/scitas-ge/scitas-stack/ci"
.parallel_definition:
parallel:
matrix:
- environment: [helvetios]
slurm_options: ["-c 36"]
stack: ["pinot-noir"]
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
path_suffix: ['']
- environment: [izar]
slurm_options: ['-c 40 --gpus 2 -p izar']
apptainer_options: ['-nv']
stack: ["pinot-noir"]
path_suffix: ['izar/']
- environment: [jed]
slurm_options: ["-c 72 -p jed -q jed"]
stack: ["pinot-noir"]
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
path_suffix: ['']
- environment: [kuma-h100]
slurm_options: ["-c 64"]
stack: ["pinot-noir"]
apptainer_options: ['--nv']
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
path_suffix: ['']
- environment: [kuma-l40s]
slurm_options: ["-c 64"]
stack: ["pinot-noir"]
apptainer_options: ['--nv']
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
path_suffix: ['']
tags:
- ${environment}
- stack
variables:
environment: ${environment}
image: ${app_image}
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
.squashfs_var:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
variables:
SQUASHFS_ID: ${CI_MERGE_REQUEST_IID}
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
SQUASHFS_ID: ${CI_COMMIT_BRANCH}
- if: $CI_COMMIT_TAG =~ /^v[0-9].*$/
variables: SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}
.parallel_job:
extends:
- .parallel_definition
- .squashfs_var
variables:
CI_DATA_ST: "/scratch/${path_suffix}scitas-stack/ci"
MOUNT_POINT: $(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
FAKEHOME: "${CI_DATA_ST}/homes"
COMMAND_OPTIONS_SBATCH: ${slurm_options}
STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
APPTAINER_EXEC_OPTIONS: >-
${apptainer_options}
--cleanenv
-H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un)
--env PYTHONUNBUFFERED=1
--bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache
--bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
--bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache
--bind ${CI_DATA_ST}/overlayfs:/overlayfs
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}"
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
spack:checkout:
stage: .pre
variables:
CI_DATA_ST: "/scratch/${path_suffix}scitas-stack/ci"
COMMAND_OPTIONS_SBATCH: ${slurm_options}
APPTAINER_EXEC_OPTIONS: >-
--cleanenv
--bind ${CI_DATA_LT}
--bind ${CI_DATA_ST}
--bind ${CI_DATA_LT}
extends:
- .parallel_definition
before_script:
- git config --global --add --bool advice.detachedHead false
script:
- ./ci/prepare_squashfs.sh
timeout: 1h
spack:setup:
stage: environment
extends:
- .parallel_job
script:
- ci/setup_spack.sh
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
needs:
- job: spack:checkout
spack:install_compilers:
stage: compilers
extends:
- .parallel_job
script:
- ci/install_compilers.sh
- source ci/stack_env.sh
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} config blame compilers
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
artifacts:
reports:
junit: spack-install-*.xml
needs:
- job: spack:setup timeout: 5h
spack:concretize:
stage: concretize
extends:
- .parallel_job
script:
- source ci/stack_env.sh
- ${STACK_LOCATION}/spack/bin/spack -e ${environment}
config blame | tee config-${environment}-${stack}.log
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
concretize | tee concretize-${environment}-${stack}.log
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
mirror create -D -d ${MOUNT_POINT}/spack-mirror -a || /usr//bin/true
- cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
artifacts:
paths:
- config-*.log
- concretize-*.log
- spack-*.lock
needs:
- job: spack:install_compilers
timeout: 24h
spack:install:
stage: install
extends:
- .parallel_job
script:
- source ci/stack_env.sh
# Cleaning patches in case one has a different hash
- ${STACK_LOCATION}/spack/bin/spack
--color always
-e ${environment}
clean -m
- ${STACK_LOCATION}/spack/bin/spack
--color always
-e ${environment} install
--log-file ${CI_PROJECT_DIR}/spack-install-${environment}.xml
--log-format junit
--only-concrete
--fail-fast
--show-log-on-error
- ${STACK_LOCATION}/spack/bin/spack
--color always
-e ${environment}
find -vl | tee spack-find-${environment}-${stack}.log
- ${STACK_LOCATION}/spack/bin/spack
--color always
-e ${environment}
buildcache create
--update-index
--key EDC904DCE3D2E84E
${MOUNT_POINT}/buildcache
- ${STACK_LOCATION}/spack/bin/spack
-e ${environment} module
lmod refresh --yes | tee spack-modules-${environment}-${stack}.log
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
needs: - job: spack:concretize
artifacts:
paths:
- spack-*.log
- spack-*.xml
reports:
junit: spack-install-*.xml
timeout: 72h
spack:mksquashfs:
stage: deploy
extends:
- .parallel_job
script:
- source ci/stack_env.sh
- mkfir -p /squashfs-cache/releases/
- mksquashfs ${STACK_LOCATION} /squashfs-cache/releases/${stack}-${environment}-${SQUASHFS_ID}-${CI_COMMIT_TAG}.sqfs
needs:
- job: spack:install
timeout: 1h
rules:
- if: $CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /v[0-9]+.*/