# yaml-language-server: $schema=gitlab-ci
# yaml-language-server: $format.enable=false

stages:
  - environment
  - compilers
  - concretize
  - install
  - release
  - deploy

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_TAG =~ /^v[0-9].*$/

variables:
  TERM: ansi
  LANG: en_US.UTF-8

.parallel_definition:
  parallel:
    matrix:
      # - environment: [helvetios]
      #   slurm_options: ["-c 36"]
      #   stack: ["pinot-noir"]
      #   app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"]
      #   path_suffix: ['']

      # - environment: [izar]
      #   slurm_options: ['-c 20 --gpus 1 -p izar']
      #   apptainer_options: ['--nv']
      #   stack: ["pinot-noir"]
      #   app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"]
      #   path_suffix: ['izar/']

      # - environment: [jed]
      #   slurm_options: ["-c 72 -p jed -q jed"]
      #   stack: ["pinot-noir"]
      #   app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"]
      #   path_suffix: ['']

      - environment: [kuma-h100]
        slurm_options: ["-c 64 -q scitas"]
        stack: ["pinot-noir"]
        apptainer_options: ['--nv']
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']

      - environment: [kuma-l40s]
        slurm_options: ["-c 64 -q scitas"]
        stack: ["pinot-noir"]
        apptainer_options: ['--nv']
        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
        path_suffix: ['']
  tags:
    - stack
  rules:
    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /v[0-9]+.*/
      variables:
        SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      variables:
        SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      variables:
        SQUASHFS_ID: ${CI_MERGE_REQUEST_IID}
  variables:
    CI_DATA_LT: "/work/scitas-ge/scitas-stack/ci"
    CI_DATA_ST: "/scratch/${path_suffix}scitas-stack/ci"
    COMMAND_OPTIONS_SBATCH: ${slurm_options}
    FAKEHOME: "${CI_DATA_ST}/homes/${stack}-${environment}-${CI_PIPELINE_ID}"
    MOUNT_POINT: $(jq -Mrc .stack.mount_point ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
    PYTHONUNBUFFERED: 1
    environment: ${environment}
    image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}

  image: ${app_image}
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
.parallel_job:
  extends:
    - .parallel_definition
  tags:
    - ${environment}
  variables:
    squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
    APPTAINER_EXEC_OPTIONS: >-
      ${apptainer_options}
         --cleanenv
         -H ${FAKEHOME}:/home/$(id -un)
         --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache
         --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
         --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache
         --bind ${CI_DATA_ST}/overlayfs:/overlayfs
         --bind ${CI_DATA_ST}/deconcretize:/deconcretize
         --fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name}-${CI_JOB_STAGE} ${squashfs_image} /overlayfs/lower-${image_name}-${CI_JOB_STAGE}"
         --fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name}-${CI_JOB_STAGE} ${stack} ${MOUNT_POINT}/${stack}"

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
spack:checkout:
  stage: .pre
  variables:
    APPTAINER_EXEC_OPTIONS: >-
      --cleanenv
      --bind ${CI_DATA_LT}
      --bind ${CI_DATA_ST}
  extends:
    - .parallel_definition
  tags:
    - ${environment}
  before_script:
    - git config --global --add --bool advice.detachedHead false
  script:
    - ./ci/prepare_squashfs.sh
    - mkdir -p ${FAKEHOME}
  timeout: 1h

spack:setup:
  stage: environment
  extends:
    - .parallel_job
  script:
    - ci/setup_spack.sh

    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  needs:
    - job: spack:checkout

spack:install_compilers:
  stage: compilers
  extends:
    - .parallel_job
  script:
    - ci/install_compilers.sh

    - source ci/stack_env.sh
    - ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} config blame compilers

    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  artifacts:
    reports:
      junit: spack-install-*.xml
  needs:
    - job: spack:setup
  timeout: 5h

spack:concretize:
  stage: concretize
  extends:
    - .parallel_job
  script:
    - ./ci/concretize.sh

    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
  artifacts:
    paths:
      - config-*.log
      - concretize-*.log
      - spack-*.lock
  needs:
    - job: spack:install_compilers
  timeout: 24h

spack:install:
  stage: install
  extends:
    - .parallel_job
  script:
    - ./ci/install.sh

    - ${CI_PROJECT_DIR}/ci/update_squashfs.sh

  needs:
    - job: spack:concretize
  artifacts:
    paths:
      - spack-*.log
      - spack-*.xml
    reports:
      junit: spack-install-*.xml
  timeout: 72h

spack:mksquashfs:
  stage: release
  extends:
    - .parallel_job
  script:
    - source ci/stack_env.sh
    - mkdir -p /squashfs-cache/releases/

    # evaluate the suffix
    - eval suffix=${release_suffix}
    - echo "${stack}-${environment}-${SQUASHFS_ID}-${suffix}.sqfs"

    - ./ci/update_squashfs.sh ${suffix}

    - mv /squashfs-cache/"${stack}-${environment}-${SQUASHFS_ID}-${suffix}.sqfs"
         /squashfs-cache/releases/
  needs:
    - job: spack:install
  timeout: 1h
  rules:
    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /v[0-9]+.*/
      variables:
        SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}
        release_suffix: ${CI_COMMIT_TAG}
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      variables:
        SQUASHFS_ID: ${CI_COMMIT_REF_SLUG}
        release_suffix: "$(date +%Y%m%d-%H%M)"

spack:deploy:
  stage: deploy
  image: registry.c4science.ch/scitas-stack/deploy
  extends:
    - .parallel_definition
  tags:
    - scitasbuild
  variables:
    MOUNT_POINT: /ssoft/spack
    COMMAND_OPTIONS_SBATCH: "-c 36"
    image_name: ${stack}-${environment}-${SQUASHFS_ID}
    squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/releases/${image_name}*.sqfs 2> /dev/null | head -1)
    APPTAINER_EXEC_OPTIONS: >-
      --bind ${MOUNT_POINT}
      --bind ${CI_DATA_LT}/squashfs-cache/
      --fusemount "container:squashfuse_ll ${squashfs_image} /squashfs"
  script:
    - ls /squashfs/${stack}/

    - mkdir -p ${MOUNT_POINT}/${stack}/
    - ls ${MOUNT_POINT}/${stack}/

    - rsync -auP /squashfs/${stack}/ ${MOUNT_POINT}/${stack}/
  needs:
    - job: spack:mksquashfs
  timeout: 2h
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

post_cleaning:
  stage: .post
  variables:
    APPTAINER_EXEC_OPTIONS: >-
      --cleanenv
      --bind ${CI_DATA_LT}
      --bind ${CI_DATA_ST}
  extends:
    - .parallel_definition
  tags:
    - ${environment}
  script:
    - ./ci/cleaning_fs.sh