apptainer_by_hand.sh

#!/usr/bin/env sh

export CI_DATA="/scratch/richart/ci"
srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"

export stack=pinot-noir
export environment=kuma_h100

export SQUASHFS_ID=local
export CI_JOB_ID=1337

export FAKEHOME="/scratch/$(id -un)"

export CI_PROJECT_DIR=$PWD
export GPG_PRIVATE_KEY=${CI_PROJECT_DIR}/stacks/buildcache.pem
export MOUNT_POINT=$(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)

APPTAINER_IMAGE=~/rhel9-kuma.sif

./ci/prepare_squashfs.sh

echo "host:/usr/libexec/apptainer/bin/squashfuse_ll ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}"
echo "host:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"

srun ${srun_options} --pty apptainer run \
    ${apptainer_options}\
    --writable-tmpfs \
    --cleanenv \
    -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \
    --bind ${CI_DATA}/buildcache:${MOUNT_POINT}/buildcache \
    --bind ${CI_DATA}/spack-mirror:${MOUNT_POINT}/spack-mirror \
    --bind ${CI_DATA}/squashfs-cache/:/squashfs-cache \
    --bind ${CI_DATA}/overlayfs:/overlayfs \
    --env stack=${stack} \
    --env environment=${environment} \
    --env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
    --env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
    --env CI_JOB_ID=${CI_JOB_ID} \
    --env SQUASHFS_ID=${SQUASHFS_ID} \
    --fusemount "host:/usr/libexec/apptainer/bin/squashfuse_ll ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}" \
    --fusemount "host:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
    ${APPTAINER_IMAGE} \
    bash