#!/usr/bin/env sh

CI_DATA="/scratch/richart/ci"
srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"

stack=pinot-noir
environment=kuma_h100

SQUASHFS_ID=local
CI_JOB_ID=1337

FAKEHOME="/scratch/$(id -un)"

CI_PROJECT_DIR=$PWD
GPG_PRIVATE_KEY=${CI_PROJECT_DIR}/stacks/buildcache.pem
MOUNT_POINT=$(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)

APPTAINER_IMAGE=~/rhel9-kuma.sif

for i in overlayfs squashfs-cache buildcache spack-mirror
do
    if [ ! -e ${CI_DATA}/$i ]
    then
        mkdir -p ${CI_DATA}/$i
    fi
done


if [ ! -f ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs ]
then
    empty=$(mktemp -d)
    mksquashfs ${empty} ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs
fi

if [ ! -f ${APPTAINER_IMAGE} ]
then
    apptainer pull -F ${APPTAINER_IMAGE}  docker://registry.c4science.ch/scitas-stack/rhel9-kuma
fi

srun ${srun_options} --pty apptainer run \
    ${apptainer_options}\
   --cleanenv \
   -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \
   --bind ${CI_DATA}/buildcache:${MOUNT_POINT}/buildcache \
   --bind ${CI_DATA}/spack-mirror:${MOUNT_POINT}/spack-mirror \
   --bind ${CI_DATA}/squashfs-cache/:/squashfs-cache \
   --bind ${CI_DATA}/overlayfs:/overlayfs \
   --env stack=${stack} \
   --env environment=${environment} \
   --env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
   --env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
   --env CI_JOB_ID=${CI_JOB_ID} \
   --env SQUASHFS_ID=${SQUASHFS_ID} \
   --fusemount "host:/usr/libexec/apptainer/bin/squashfuse_ll ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}" \
   --fusemount "host:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
   ${APPTAINER_IMAGE}
   bash
