Moving code around to simplify the "by_hand" version

.gitlab-ci.yml

@@ -97,7 +97,8 @@ variables:
          --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
          --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache
          --bind ${CI_DATA_ST}/overlayfs:/overlayfs
-         --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}"
+         --bind ${CI_DATA_ST}/deconcretize:/deconcretize
+         --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name}-${CI_JOB_STAGE} ${squashfs_image} /overlayfs/lower-${image_name}"
          --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
 
 # ------------------------------------------------------------------------------
@@ -124,6 +125,7 @@ spack:setup:
     - .parallel_job
   script:
     - ci/setup_spack.sh
+
     - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
   needs:
     - job: spack:checkout
@@ -149,29 +151,8 @@ spack:concretize:
   extends:
     - .parallel_job
   script:
-    - source ci/stack_env.sh
-
-    - echo "Checking ${CI_DATA_ST}/deconcretize_${stack}_${environment}_${SQUASHFS_ID}"
-    - |
-      if [ -e ${CI_DATA_ST}/deconcretize_${stack}_${environment}_${SQUASHFS_ID} ];
-      then
-         while read -r spec
-         do
-           echo "Deconcretizing ${spec}"
-           ${STACK_LOCATION}/spack/bin/spack --color=always -e ${environment} deconcretize --all ${spec}
-         done <<< $(cat ${CI_DATA_ST}/deconcretize_${stack}_${environment}_${SQUASHFS_ID})
-      fi
-
-    - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
-          config blame | tee config-${environment}-${stack}.log
-
-    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
-          concretize | tee concretize-${environment}-${stack}.log
+    - ./ci/concretize.sh
 
-    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
-          mirror create -D -d ${MOUNT_POINT}/spack-mirror -a || /usr//bin/true
-
-    - cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock
     - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
   artifacts:
     paths:
@@ -187,41 +168,10 @@ spack:install:
   extends:
     - .parallel_job
   script:
-    - source ci/stack_env.sh
-
-    # Cleaning patches in case one has a different hash
-    - ${STACK_LOCATION}/spack/bin/spack
-          --color always
-          -e ${environment}
-          clean -m
-
-    - ${STACK_LOCATION}/spack/bin/spack
-          --color always
-          -e ${environment} install
-          --log-file ${CI_PROJECT_DIR}/spack-install-${environment}.xml
-          --log-format junit
-          --only-concrete
-          --fail-fast
-          --show-log-on-error
-
-    - ${STACK_LOCATION}/spack/bin/spack
-          --color always
-          -e ${environment}
-          find -vl | tee spack-find-${environment}-${stack}.log
-
-    - ${STACK_LOCATION}/spack/bin/spack
-          --color always
-          -e ${environment}
-          buildcache create
-          --update-index
-          --key ${GPG_KEY_ID}
-          ${MOUNT_POINT}/buildcache
-
-    - ${STACK_LOCATION}/spack/bin/spack
-          -e ${environment} module
-          lmod refresh --yes | tee spack-modules-${environment}-${stack}.log
+    - ./ci/install.sh
 
     - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
+
   needs:
     - job: spack:concretize
   artifacts:

ci/concretize.sh

+#!/usr/bin/env sh
+
+source ci/stack_env.sh
+
+echo "Checking /deconcretize/${stack}_${environment}_${SQUASHFS_ID}"
+if [ -e /deconcretize/${stack}_${environment}_${SQUASHFS_ID} ];
+then
+   while read -r spec
+   do
+     echo "Deconcretizing ${spec}"
+     ${STACK_LOCATION}/spack/bin/spack --color=always -e ${environment} deconcretize --all ${spec}
+   done <<< $(cat /deconcretize/${stack}_${environment}_${SQUASHFS_ID})
+fi
+
+${STACK_LOCATION}/spack/bin/spack \
+    -e ${environment} \
+    config blame | tee config-${environment}-${stack}.log
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} \
+    concretize | tee concretize-${environment}-${stack}.log
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} \
+    mirror create \
+    -D -d ${MOUNT_POINT}/spack-mirror -a || /usr//bin/true
+
+cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock

ci/install.sh

+#!/usr/bin/env sh
+
+source ci/stack_env.sh
+
+# Cleaning patches in case one has a different hash
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} \
+    clean -m
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} install \
+    --log-file ${CI_PROJECT_DIR}/spack-install-${environment}.xml \
+    --log-format junit \
+    --only-concrete \
+    --fail-fast \
+    --show-log-on-error
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} \
+    find -vl | tee spack-find-${environment}-${stack}.log
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} \
+    buildcache create \
+    --update-index \
+    --key ${GPG_KEY_ID} \
+    ${MOUNT_POINT}/buildcache
+
+${STACK_LOCATION}/spack/bin/spack \
+    --color always \
+    -e ${environment} module \
+    lmod refresh --yes | tee spack-modules-${environment}-${stack}.log

ci/prepare_squashfs.sh

@@ -7,6 +7,7 @@ source ${CI_PROJECT_DIR}/ci/stack_env.sh
 
 mkdir -p ${CI_DATA_ST}/overlayfs
 mkdir -p ${CI_DATA_ST}/homes
+mkdir -p ${CI_DATA_ST}/deconcretize
 
 for i in squashfs-cache buildcache spack-mirror
 do

ci/squashfuse_ll.sh

@@ -10,10 +10,10 @@ mount_point="$*"
 
 for i in upper wd
 do
-    if [ -d ${CI_DATA}/overlayfs/$i-${suffix} ]
-    then
-        rm -rf ${CI_DATA}/overlayfs/$i-${suffix}
-    fi
+    # if [ -d ${CI_DATA}/overlayfs/$i-${suffix} ]
+    # then
+    #     rm -rf ${CI_DATA}/overlayfs/$i-${suffix}
+    # fi
     mkdir -p ${CI_DATA}/overlayfs/$i-${suffix}
 done