Update of CI tu use squashfs in CI to build MR on existing stacks

.gitlab-ci.yml

@@ -17,6 +17,7 @@ workflow:
 
 variables:
   TERM: ansi
+  CI_DATA: "/work/scitas-ge/richart/ci"
 
 .parallel_definition:
   parallel:
@@ -55,12 +56,18 @@ variables:
   extends:
     - .parallel_definition
   variables:
-    MOUNT_POINT: /ssoft/spack
-    BUILDCACHE: "/work/scitas-ge/richart/ci/buildcache"
-    MIRROR: "/work/scitas-ge/richart/ci/mirror"
-    FAKEHOME: "/scratch/richart"
+    MOUNT_POINT: $(jq -Mc .stack.mount_point stacks/${stack}/config.json)
+    FAKEHOME: "/scratch/$(id -un)"
     COMMAND_OPTIONS_SBATCH: ${slurm_options}
-    APPTAINER_EXEC_OPTIONS: ${apptainer_options} --cleanenv -H $(mktemp -d -p ${FAKEHOME}/):/home/richart --bind ${BUILDCACHE}:/buildcache:rw --bind ${MIRROR}:${MOUNT_POINT}/spack-mirror:rw --bind ${CI_PROJECT_DIR}:${MOUNT_POINT} --bind /dev/tty
+    STACK_VERSION: $(jq -Mc .stack.version stacks/${stack}/config.json)
+    APPTAINER_EXEC_OPTIONS: >- ${apptainer_options} --cleanenv
+      -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un)
+      --bind ${CI_DATA}/buildcache:${MOUNT_POINT}/buildcache
+      --bind ${CI_DATA}/spack-mirror:${MOUNT_POINT}/spack-mirror
+      --bind ${CI_DATA}/squashfs-cache/:/squashfs-cache \
+      --bind ${CI_DATA}/overlayfs:/overlayfs \
+      --fusermount "host:/usr/libexec/apptainer/bin/squashfuse_ll ${CI_DATA}/squashfs-cache/${stack}-${environment}-${CI_MERGE_REQUEST_IID}.sqfs /squashfs"
+      --fusermount "conqtainer:fuse-overlayfs -o squash_to_uid=$(id -n) -o squash_to_gid=$(id -g) -o lowerdir=/squashfs -o upperdir=/overlayfs/upper -o workdir=/overlayfs/wd ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
 
 .spack_cache:
   cache:
@@ -73,12 +80,18 @@ variables:
 # ------------------------------------------------------------------------------
 spack:checkout:
   stage: .pre
+  variables:
+    APPTAINER_EXEC_OPTIONS: --cleanenv --bind ${CI_DATA}
   extends:
     - .parallel_definition
   before_script:
     - git config --global --add --bool advice.detachedHead false
   script:
-    - ls
+    - mkdir -p ${CI_DATA}/overlayfs
+    - mkdir -p ${CI_DATA}/squashfs-cache
+    - mkdir -p ${CI_DATA}/buildcache
+    - mkdir -p ${CI_DATA}/spack-mirror
+    - ./ci/prepare_squashfs.sh
   timeout: 1h
 
 spack:setup:
@@ -114,8 +127,12 @@ spack:concretize:
     - .spack_cache
   script:
     - source ci/stack_env.sh
-    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame | tee config-${environment}-${stack}.log
-    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize | tee concretize-${environment}-${stack}.log
+
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
+          config blame | tee config-${environment}-${stack}.log
+
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
+          concretize | tee concretize-${environment}-${stack}.log
 
     - cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock
   artifacts:
@@ -137,18 +154,18 @@ spack:install:
     - echo "{}" > spack-install-${environment}.xml
 
     - ${STACK_LOCATION}/spack/bin/spack -e ${environment} install
-      --log-file spack-install-${environment}.xml
-      --log-format junit
-      --only-concrete
-      --fail-fast
-      --show-log-on-error
+          --log-file spack-install-${environment}.xml
+          --log-format junit
+          --only-concrete
+          --fail-fast
+          --show-log-on-error
 
     - ${STACK_LOCATION}/spack/bin/spack -e ${environment} find -vl | tee spack-find-${environment}-${stack}.log
     - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
-      buildcache create
-      --update-index
-      --key EDC904DCE3D2E84E
-      /buildcache
+          buildcache create
+          --update-index
+          --key EDC904DCE3D2E84E
+          ${MOUNT_POINT}/buildcache
   needs:
     - job: spack:concretize
   artifacts:

ci/install_compilers.sh

@@ -53,7 +53,7 @@ for c in  "core_compilers" "compilers"; do
         ${STACK_LOCATION}/spack/bin/spack buildcache create \
             --update-index \
             --key EDC904DCE3D2E84E \
-            /buildcache ${hashes}
+            ${MOUNT_POINT}/buildcache ${hashes}
 
         for compiler_hash in $(echo ${hashes}); do
             location=$(${STACK_LOCATION}/spack/bin/spack location -i ${compiler_hash})

ci/prepare_squashfs.sh

+#!/usr/bin/env sh
+
+set -o pipefail
+
+
+source ${CI_PROJECT_DIR}/ci/stack_env.sh
+
+if [ "x${CI_DEFAULT_BRANCH}" != "x" ]
+then
+   squash_base=${CI_DEFAULT_BRANCH}
+else
+   squash_base=master
+fi
+
+if [ "x${CI_MERGE_REQUEST_IID}" != "x" ]
+then
+    squash_id=${CI_MERGE_REQUEST_IID}
+elif [ "x${CI_COMMIT_BRANCH}" != "x" ]
+then
+    squash_id=${CI_MERGE_REQUEST_IID}
+else
+    squash_id="local"
+fi
+
+set -o nounset
+
+if [ -d /squashfs-cache ]
+then
+    mkdir /squashfs-cache
+fi
+
+# Check if MR as squashfs
+sqfs_image=$(ls -t1 /squashfs-cache/${stack}-${environment}-${squash_id}.sqfs 2> /dev/null | head -1)
+if [ $? -ne 0 ]
+then
+    echo "No MR ($squash_id) squashfs found"
+    # Check if MR default branch as squashfs
+    sqfs_base_image=$(ls -t1 /squashfs-cache/${stack}-${environment}-${squash_base}.sqfs 2> /dev/null | head -1)
+    if [ $? -ne 0 ]
+    then
+        echo "No default branch ($squash_base) squashfs found"
+        echo "Creating an empty one"
+        mkdir /tmp/empty
+        mksquashfs /tmp/empty /squashfs-cache/${stack}-${environment}-${squash_base}.sqfs
+    else
+        echo "Found ${sqfs_base_image}"
+    fi
+
+    # Link MR sqaushfs to the one of default branch
+    ln -sf /squashfs-cache/${stack}-${environment}-${squash_base}.sqfs \
+        /squashfs-cache/${stack}-${environment}-${squash_id}.sqfs
+
+    echo "Linking ${stack}-${environment}-${squash_id}.sqfs -> ${stack}-${environment}-${squash_base}.sqfs"
+else
+    echo "Found ${sqfs_image}"
+fi
+
+
+if [ -d /overlayfs ]
+then
+    mkdir -p /overlayfs
+fi
+
+if [ -d /overlayfs/upper ]
+then
+    mkdir -p /overlayfs/upper
+    mkdir -p /overlayfs/wd
+fi

ci/setup_spack.sh

@@ -147,9 +147,9 @@ echo "Setting up buildcache"
 spack/bin/spack gpg trust \
     $GPG_PRIVATE_KEY
 
-if [ ! -d /buildcache/build_cache ]; then
+if [ ! -d ${MOUNT_POINT}/buildcache/build_cache ]; then
   spack/bin/spack gpg publish \
-      -d /buildcache
+      -d ${MOUNT_POINT}/buildcache
 fi
 
 if ! spack/bin/spack mirror list | grep buildcache; then
@@ -157,7 +157,7 @@ if ! spack/bin/spack mirror list | grep buildcache; then
   spack/bin/spack mirror add \
       --type binary \
       --scope system \
-      buildcache file:///buildcache
+      buildcache file://${MOUNT_POINT}/buildcache
 fi
 
 spack/bin/spack buildcache keys \

ci/stack_env.sh

@@ -10,8 +10,8 @@ export STACK_LOCATION=${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}
 
 export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/spack/var/spack/environments/${environment}
 
-export SPACK_USER_CACHE_PATH=$(mktemp -p /tmp -d slurm_user_cache_XXXXXXX)
-export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)
+#export SPACK_USER_CACHE_PATH=$(mktemp -p /tmp -d slurm_user_cache_XXXXXXX)
+#export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)
 
 export environment_type="local_cluster"
 
@@ -21,5 +21,5 @@ echo "SPACK_VERSION: ${SPACK_VERSION}"
 echo "MOUNT_POINT: ${MOUNT_POINT}"
 echo "STACK_LOCATION: ${STACK_LOCATION}"
 echo "SPACK_SYSTEM_CONFIG_PATH: ${SPACK_SYSTEM_CONFIG_PATH}"
-echo "SPACK_USER_CACHE_PATH: ${SPACK_USER_CACHE_PATH}"
-echo "SPACK_USER_CONFIG_PATH: ${SPACK_USER_CONFIG_PATH}"
+# echo "SPACK_USER_CACHE_PATH: ${SPACK_USER_CACHE_PATH}"
+# echo "SPACK_USER_CONFIG_PATH: ${SPACK_USER_CONFIG_PATH}"