apptainer_by_hand.sh

#!/usr/bin/env sh

export CI_DATA_LT="/work/scitas-ge/richart/ci"
export CI_DATA_ST="/scratch/richart/ci"
srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"

export stack=pinot-noir
export environment=kuma_h100

export SQUASHFS_ID=local
export CI_PIPELINE_ID=1337

export FAKEHOME="/scratch/$(id -un)"

export CI_PROJECT_DIR=$PWD
export GPG_PRIVATE_KEY=${CI_PROJECT_DIR}/stacks/buildcache.pem
export MOUNT_POINT=$(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)

APPTAINER_IMAGE=~/rhel9-kuma.sif
image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
#./ci/prepare_squashfs.sh

srun ${srun_options} --pty apptainer run \
    ${apptainer_options}\
    --writable-tmpfs \
    --cleanenv \
    -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \
    --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache \
    --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror \
    --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache \
    --bind ${CI_DATA_ST}/overlayfs:/overlayfs \
    --env stack=${stack} \
    --env environment=${environment} \
    --env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
    --env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
    --env CI_JOB_ID=${CI_JOB_ID} \
    --env SQUASHFS_ID=${SQUASHFS_ID} \
    --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \
    --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
    ${APPTAINER_IMAGE} \
    bash