From 31a5b80856c0760109619a1b9f8b56644604c29c Mon Sep 17 00:00:00 2001
From: Nicolas Richart <nicolas.richart@epfl.ch>
Date: Tue, 23 Jul 2024 15:29:35 +0200
Subject: [PATCH] Adding deploy step to keep everythng per MR
---
.gitlab-ci.yml | 13 ++++++-------
ci/apptainer_by_hand.sh | 6 ++++--
ci/prepare_squashfs.sh | 27 ++++++++++++++++++---------
3 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98c022e..e11fe4b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,7 +71,7 @@ variables:
FAKEHOME: "/scratch/$(id -un)"
COMMAND_OPTIONS_SBATCH: ${slurm_options}
STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
- image_name: "${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}"
+ image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
APPTAINER_EXEC_OPTIONS: >-
${apptainer_options}
@@ -210,11 +210,10 @@ spack:mksquashfs:
script:
- source ci/stack_env.sh
- - echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
- - "mksquashfs ${STACK_LOCATION} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
- - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
+ - mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${SQUASH_ID}.sqfs
+ #- echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
+ #- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
needs:
- job: spack:install
- timeout: 10h
- rules:
- - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+ ressource_group: mksquashfs
+ timeout: 1h
diff --git a/ci/apptainer_by_hand.sh b/ci/apptainer_by_hand.sh
index ba95ef6..131cfbf 100755
--- a/ci/apptainer_by_hand.sh
+++ b/ci/apptainer_by_hand.sh
@@ -20,8 +20,10 @@ export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/
APPTAINER_IMAGE=~/rhel9-kuma.sif
image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
+
+./ci/prepare_squashfs.sh
+
squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
-#./ci/prepare_squashfs.sh
srun ${srun_options} --pty apptainer run \
${apptainer_options}\
@@ -36,7 +38,7 @@ srun ${srun_options} --pty apptainer run \
--env environment=${environment} \
--env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
--env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
- --env CI_JOB_ID=${CI_JOB_ID} \
+ --env CI_PIPELINE_ID=${CI_PIPELINE_ID} \
--env SQUASHFS_ID=${SQUASHFS_ID} \
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
diff --git a/ci/prepare_squashfs.sh b/ci/prepare_squashfs.sh
index ed5f5b3..b51e4b9 100755
--- a/ci/prepare_squashfs.sh
+++ b/ci/prepare_squashfs.sh
@@ -21,20 +21,29 @@ set +o errexit
sqfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
- echo "No MR ($squash_id) squashfs found"
- # Check if MR default branch as squashfs
- sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
+ echo "No MR ($squash_id-${CI_PIPELINE_ID}) squashfs found"
+
+ # look for base in the branch
+ sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
- echo "No default branch ($squash_base) squashfs found"
- echo "Creating an empty one"
- empty=$(mktemp -d)
- sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
- mksquashfs ${empty} ${sqfs_base_image}
+ echo "No MR ($squash_id) squashfs found"
+
+ # look for base in default branch
+ sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
+ if [ $? -ne 0 ]
+ then
+ echo "No default branch ($squash_base) squashfs found"
+ echo "Creating an empty one"
+ empty=$(mktemp -d)
+ sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
+ mksquashfs ${empty} ${sqfs_base_image}
+ else
+ echo "Found ${sqfs_base_image}"
+ fi
else
echo "Found ${sqfs_base_image}"
fi
-
set -o errexit
# Link MR sqaushfs to the one of default branch
--
GitLab