Adding deploy step to keep everythng per MR

.gitlab-ci.yml

@@ -71,7 +71,7 @@ variables:
     FAKEHOME: "/scratch/$(id -un)"
     COMMAND_OPTIONS_SBATCH: ${slurm_options}
     STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
-    image_name: "${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}"
+    image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
     squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
     APPTAINER_EXEC_OPTIONS: >-
       ${apptainer_options}
@@ -210,11 +210,10 @@ spack:mksquashfs:
   script:
     - source ci/stack_env.sh
 
-    - echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
-    - "mksquashfs ${STACK_LOCATION} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
-    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
+    - mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${SQUASH_ID}.sqfs
+    #- echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
+    #- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
   needs:
     - job: spack:install
-  timeout: 10h
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  ressource_group: mksquashfs
+  timeout: 1h

ci/apptainer_by_hand.sh

@@ -20,8 +20,10 @@ export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/
 
 APPTAINER_IMAGE=~/rhel9-kuma.sif
 image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
+
+./ci/prepare_squashfs.sh
+
 squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
-#./ci/prepare_squashfs.sh
 
 srun ${srun_options} --pty apptainer run \
     ${apptainer_options}\
@@ -36,7 +38,7 @@ srun ${srun_options} --pty apptainer run \
     --env environment=${environment} \
     --env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
     --env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
-    --env CI_JOB_ID=${CI_JOB_ID} \
+    --env CI_PIPELINE_ID=${CI_PIPELINE_ID} \
     --env SQUASHFS_ID=${SQUASHFS_ID} \
     --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \
     --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \

ci/prepare_squashfs.sh

@@ -21,20 +21,29 @@ set +o errexit
 sqfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}.sqfs 2> /dev/null | head -1)
 if [ $? -ne 0 ]
 then
-    echo "No MR ($squash_id) squashfs found"
-    # Check if MR default branch as squashfs
-    sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
+    echo "No MR ($squash_id-${CI_PIPELINE_ID}) squashfs found"
+
+    # look for base in the branch
+    sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs 2> /dev/null | head -1)
     if [ $? -ne 0 ]
     then
-        echo "No default branch ($squash_base) squashfs found"
-        echo "Creating an empty one"
-        empty=$(mktemp -d)
-        sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
-        mksquashfs ${empty} ${sqfs_base_image}
+        echo "No MR ($squash_id) squashfs found"
+
+        # look for base in default branch
+        sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
+        if [ $? -ne 0 ]
+        then
+            echo "No default branch ($squash_base) squashfs found"
+            echo "Creating an empty one"
+            empty=$(mktemp -d)
+            sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
+            mksquashfs ${empty} ${sqfs_base_image}
+        else
+            echo "Found ${sqfs_base_image}"
+        fi
     else
         echo "Found ${sqfs_base_image}"
     fi
-
     set -o errexit
 
     # Link MR sqaushfs to the one of default branch