From e692c698b11852c3ab70968f7ee1fc254e326599 Mon Sep 17 00:00:00 2001
From: Nicolas Richart <nicolas.richart@epfl.ch>
Date: Tue, 23 Jul 2024 11:58:12 +0200
Subject: [PATCH] Over complicating things because /work is slow and /scratch
is not safe
---
.gitlab-ci.yml | 22 ++++++++++++++--------
ci/apptainer_by_hand.sh | 20 +++++++++-----------
ci/prepare_squashfs.sh | 20 ++++++++++----------
ci/squashfuse_ll.sh | 4 +++-
4 files changed, 36 insertions(+), 30 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 076e2fd..3c0080f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,7 +17,8 @@ workflow:
variables:
TERM: ansi
- CI_DATA: "/work/scitas-ge/richart/ci"
+ CI_DATA_LT: "/work/scitas-ge/richart/ci"
+ CI_DATA_ST: "/scratch/richart/ci"
.parallel_definition:
parallel:
@@ -70,16 +71,17 @@ variables:
FAKEHOME: "/scratch/$(id -un)"
COMMAND_OPTIONS_SBATCH: ${slurm_options}
STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
+ image_name: "${stack}-${environment}-${SQUASHFS_ID}"
APPTAINER_EXEC_OPTIONS: >-
${apptainer_options}
--cleanenv
-H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un)
- --bind ${CI_DATA}/buildcache:${MOUNT_POINT}/buildcache
- --bind ${CI_DATA}/spack-mirror:${MOUNT_POINT}/spack-mirror
- --bind ${CI_DATA}/squashfs-cache/:/squashfs-cache
- --bind ${CI_DATA}/overlayfs:/overlayfs
- --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}"
- --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
+ --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache
+ --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
+ --bind ${CI_DATA_ST}/squashfs-cache/:/squashfs-cache
+ --bind ${CI_DATA_ST}/overlayfs:/overlayfs
+ --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${CI_DATA_LT}/squashfs-cache /overlayfs/lower-${image_name}"
+ --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
# after_script:
# - if [ $CI_JOB_STATUS != "success" ]; then exit 0; fi
# - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
@@ -96,7 +98,11 @@ variables:
spack:checkout:
stage: .pre
variables:
- APPTAINER_EXEC_OPTIONS: --cleanenv --bind ${CI_DATA} --bind ${CI_DATA}/squashfs-cache:/squashfs-cache
+ APPTAINER_EXEC_OPTIONS: >-
+ --cleanenv
+ --bind ${CI_DATA_LT}
+ --bind ${CI_DATA_ST}
+ --bind ${CI_DATA_LT}/squashfs-cache:/squashfs-cache
extends:
- .parallel_definition
before_script:
diff --git a/ci/apptainer_by_hand.sh b/ci/apptainer_by_hand.sh
index 7b7d41d..a6ca885 100755
--- a/ci/apptainer_by_hand.sh
+++ b/ci/apptainer_by_hand.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env sh
-export CI_DATA="/scratch/richart/ci"
+export CI_DATA_LT="/work/scitas-ge/richart/ci"
+export CI_DATA_ST="/scratch/richart/ci"
srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"
@@ -18,28 +19,25 @@ export MOUNT_POINT=$(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack
export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
APPTAINER_IMAGE=~/rhel9-kuma.sif
-
+image_name=${stack}-${environment}-${SQUASHFS_ID}
#./ci/prepare_squashfs.sh
-echo "host:/usr/libexec/apptainer/bin/squashfuse_ll ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}"
-echo "host:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
-
srun ${srun_options} --pty apptainer run \
${apptainer_options}\
--writable-tmpfs \
--cleanenv \
-H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \
- --bind ${CI_DATA}/buildcache:${MOUNT_POINT}/buildcache \
- --bind ${CI_DATA}/spack-mirror:${MOUNT_POINT}/spack-mirror \
- --bind ${CI_DATA}/squashfs-cache/:/squashfs-cache \
- --bind ${CI_DATA}/overlayfs:/overlayfs \
+ --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache \
+ --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror \
+ --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache \
+ --bind ${CI_DATA_ST}/overlayfs:/overlayfs \
--env stack=${stack} \
--env environment=${environment} \
--env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
--env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
--env CI_JOB_ID=${CI_JOB_ID} \
--env SQUASHFS_ID=${SQUASHFS_ID} \
- --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA} ${SQUASHFS_ID}-${CI_JOB_ID} ${CI_DATA}/squashfs-cache/${stack}-${environment}-${SQUASHFS_ID}.sqfs /overlayfs/lower-${SQUASHFS_ID}-${CI_JOB_ID}" \
- --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${SQUASHFS_ID}-${CI_JOB_ID} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
+ --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${CI_DATA_LT}/squashfs-cache /overlayfs/lower-${image_name}" \
+ --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
${APPTAINER_IMAGE} \
bash
diff --git a/ci/prepare_squashfs.sh b/ci/prepare_squashfs.sh
index be78be6..9f2fee1 100755
--- a/ci/prepare_squashfs.sh
+++ b/ci/prepare_squashfs.sh
@@ -5,29 +5,29 @@ set -o nounset
set -o errexit
source ${CI_PROJECT_DIR}/ci/stack_env.sh
-for i in overlayfs squashfs-cache buildcache spack-mirror
+for i in squashfs-cache buildcache spack-mirror
do
- if [ ! -e ${CI_DATA}/$i ]
+ if [ ! -e ${CI_DATA_LT}/$i ]
then
- mkdir -p ${CI_DATA}/$i
+ mkdir -p ${CI_DATA_LT}/$i
fi
done
set +o errexit
# Check if MR as squashfs
-sqfs_image=$(ls -t1 ${CI_DATA}/squashfs-cache/${stack}-${environment}-${squash_id}*.sqfs 2> /dev/null | head -1)
+sqfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}*.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
echo "No MR ($squash_id) squashfs found"
# Check if MR default branch as squashfs
- sqfs_base_image=$(ls -t1 ${CI_DATA}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
+ sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
echo "No default branch ($squash_base) squashfs found"
echo "Creating an empty one"
empty=$(mktemp -d)
- sqfs_base_image="${CI_DATA}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
+ sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
mksquashfs ${empty} ${sqfs_base_image}
else
echo "Found ${sqfs_base_image}"
@@ -36,18 +36,18 @@ then
set -o errexit
# Link MR sqaushfs to the one of default branch
- sqfs_image=${CI_DATA}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs
+ sqfs_image=${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs
- cd ${CI_DATA}/squashfs-cache
+ cd ${CI_DATA_LT}/squashfs-cache
ln -sf $(basename ${sqfs_base_image}) $(basename ${sqfs_image})
cd -
echo "Linking ${sqfs_image} -> ${sqfs_base_image}"
else
set -o errexit
- if [ ${sqfs_image} != ${CI_DATA}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs ];
+ if [ ${sqfs_image} != ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs ];
then
- cd ${CI_DATA}/squashfs-cache
+ cd ${CI_DATA_LT}/squashfs-cache
ln -sf $(basname ${sqfs_image}) ${stack}-${environment}-${squash_id}.sqfs
cd
fi
diff --git a/ci/squashfuse_ll.sh b/ci/squashfuse_ll.sh
index 7e22cbc..368ae14 100755
--- a/ci/squashfuse_ll.sh
+++ b/ci/squashfuse_ll.sh
@@ -2,7 +2,7 @@
CI_DATA=$1
suffix=$2
-image=$3
+image_path=$3
mount_point=$4
for i in upper wd
@@ -14,4 +14,6 @@ do
mkdir -p ${CI_DATA}/overlayfs/$i-${suffix}
done
+image=${image_path}/${suffix}.sqfs
+
/usr/libexec/apptainer/bin/squashfuse_ll $image $mount_point
--
GitLab