Skip to content
Snippets Groups Projects

Adding nvhpc to kuma

Merged Nicolas Richart requested to merge adding-nvhpc into main
Compare and Show latest version
11 files
+ 83
69
Compare changes
  • Side-by-side
  • Inline
Files
11
+ 25
10
#!/usr/bin/env sh
export stack=pinot-noir
export environment=kuma-l40s
declare -A slurm_options
slurm_options["helvetios"]="-N1 -n1 -c 36"
slurm_options["izar"]="-c 40 --gpus 2 -p izar"
slurm_options["jed"]="-N1 -n1 -c 72 -p jed -q jed"
slurm_options["kuma-l40s"]="-N1 -n1 -c 64 -q scitas -p l40s"
slurm_options["kuma-h100"]="-N1 -n1 -c 64 -q scitas -p h100"
declare -A apptainer_options
apptainer_options["helvetios"]=""
apptainer_options["izar"]="--nv"
apptainer_options["jed"]=""
apptainer_options["kuma-l40s"]="--nv"
apptainer_options["kuma-h100"]="--nv"
export CI_DATA_LT="/work/scitas-ge//$(id -un)/ci"
export CI_DATA_ST="/scratch//$(id -un)/ci"
srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"
export stack=pinot-noir
export environment=kuma-h100
export SQUASHFS_ID=local
export CI_PIPELINE_ID=1337
export GPG_KEY_ID=EDC904DCE3D2E84E
export FAKEHOME="${CI_DATA_ST}/homes"
export CI_PROJECT_DIR=$PWD
@@ -19,14 +33,14 @@ export MOUNT_POINT=$(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack
export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
APPTAINER_IMAGE=~/rhel9-kuma.sif
image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
export image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
./ci/prepare_squashfs.sh
squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
srun ${srun_options} --pty apptainer run \
${apptainer_options}\
srun ${slurm_options[$environment]} --time 1-0:0:0 --pty apptainer run \
${apptainer_options[$environment]} \
--writable-tmpfs \
--cleanenv \
-H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \
@@ -40,12 +54,13 @@ srun ${srun_options} --pty apptainer run \
--env stack=${stack} \
--env environment=${environment} \
--env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
--env GPG_KEY_ID=${GPG_KEY_ID} \
--env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
--env CI_PIPELINE_ID=${CI_PIPELINE_ID} \
--env CI_DATA_LT=${CI_DATA_LT} \
--env CI_DATA_ST=${CI_DATA_ST} \
--env SQUASHFS_ID=${SQUASHFS_ID} \
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} ${stack} /overlayfs/lower-${image_name}" \
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${stack} ${MOUNT_POINT}/${stack}" \
${APPTAINER_IMAGE} \
bash
Loading