diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98c022e6e9c4257a0b379f470fcfbc96a46111f8..e11fe4b39d047524d6512282ef97bcca1d4ef3f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,7 +71,7 @@ variables: FAKEHOME: "/scratch/$(id -un)" COMMAND_OPTIONS_SBATCH: ${slurm_options} STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json) - image_name: "${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}" + image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID} squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1) APPTAINER_EXEC_OPTIONS: >- ${apptainer_options} @@ -210,11 +210,10 @@ spack:mksquashfs: script: - source ci/stack_env.sh - - echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs" - - "mksquashfs ${STACK_LOCATION} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs" - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"' + - mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${SQUASH_ID}.sqfs + #- echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs" + #- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"' needs: - job: spack:install - timeout: 10h - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + ressource_group: mksquashfs + timeout: 1h diff --git a/ci/apptainer_by_hand.sh b/ci/apptainer_by_hand.sh index ba95ef61b8d7e50ade9b205f9c4fe81aaf438cfc..131cfbf819812501aae905eb20b14fa89530de88 100755 --- a/ci/apptainer_by_hand.sh +++ b/ci/apptainer_by_hand.sh @@ -20,8 +20,10 @@ export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/ APPTAINER_IMAGE=~/rhel9-kuma.sif image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID} + +./ci/prepare_squashfs.sh + squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1) -#./ci/prepare_squashfs.sh srun ${srun_options} --pty apptainer run \ ${apptainer_options}\ @@ -36,7 +38,7 @@ srun ${srun_options} --pty apptainer run \ --env environment=${environment} \ --env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \ --env CI_PROJECT_DIR=${CI_PROJECT_DIR} \ - --env CI_JOB_ID=${CI_JOB_ID} \ + --env CI_PIPELINE_ID=${CI_PIPELINE_ID} \ --env SQUASHFS_ID=${SQUASHFS_ID} \ --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \ --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \ diff --git a/ci/prepare_squashfs.sh b/ci/prepare_squashfs.sh index ed5f5b33c97c7551cb7fc5301b8dc76a86947e51..b51e4b97beb3d65ee44336dab56f963e4ec9f50b 100755 --- a/ci/prepare_squashfs.sh +++ b/ci/prepare_squashfs.sh @@ -21,20 +21,29 @@ set +o errexit sqfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}.sqfs 2> /dev/null | head -1) if [ $? -ne 0 ] then - echo "No MR ($squash_id) squashfs found" - # Check if MR default branch as squashfs - sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1) + echo "No MR ($squash_id-${CI_PIPELINE_ID}) squashfs found" + + # look for base in the branch + sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs 2> /dev/null | head -1) if [ $? -ne 0 ] then - echo "No default branch ($squash_base) squashfs found" - echo "Creating an empty one" - empty=$(mktemp -d) - sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs" - mksquashfs ${empty} ${sqfs_base_image} + echo "No MR ($squash_id) squashfs found" + + # look for base in default branch + sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1) + if [ $? -ne 0 ] + then + echo "No default branch ($squash_base) squashfs found" + echo "Creating an empty one" + empty=$(mktemp -d) + sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs" + mksquashfs ${empty} ${sqfs_base_image} + else + echo "Found ${sqfs_base_image}" + fi else echo "Found ${sqfs_base_image}" fi - set -o errexit # Link MR sqaushfs to the one of default branch