.gitlab-ci.yml

# yaml-language-server: $schema=gitlab-ci
# yaml-language-server: $format.enable=false

stages:
  - environment
  - compilers
  - concretize
  - install
  - deploy

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

variables:
  TERM: ansi

# ------------------------------------------------------------------------------
.parallel_job:
  image: registry.c4science.ch/scitas-stack/rhel:9.3
  parallel:
    matrix:
#      - environment: [helvetios]
#        slurm_options: ["-c 36"]
      - environment: [jed]
        slurm_options: ["-c 72 -p jed -q jed"]
      # - environment: [izar]
      #   slurm_options: ['-c 40 --gres gpu:2']
      #   apptainer_options: ['-nv']
  tags:
    - ${environment}
    - stack
  variables:
    COMMAND_OPTIONS_SBATCH: ${slurm_options}
    APPTAINER_EXEC_OPTIONS: ${apptainer_options} --bind ${BUILDCACHE}:/buildcache:rw --bind ${STACK_PATH}:/stack:rw --writable-tmpfs
    SPACK_DISABLE_LOCAL_CONFIG: "true"
    SPACK_USER_CACHE_PATH: "/tmp/spack_cache_${CI_JOB_ID}"
    SPACK_SITE_CONFIG_PATH: "/stack/spack/etc"
    BUILDCACHE: "/work/scitas-ge/richart/ci/buildcache"
    STACK_PATH: "/work/scitas-ge/richart/ci/${environment}/stack"


.spack_cache:
  cache:
    key: ${environment}-spack-cache
    policy: pull-push
    paths:
      - spack/
      - compiler_definitions.yaml

# ------------------------------------------------------------------------------
spack:setup:
  stage: environment
  extends:
    - .spack_cache
    - .parallel_job
  before_script:
    - git config --global --add --bool advice.detachedHead false
  script:
    - SPACK_VERSION=$(jq -r .spack.version config.json)

    - |
      if [ ! -d spack/.git ]; then
        git clone https://github.com/spack/spack.git -b $SPACK_VERSION spack
      else
        git -C spack fetch
        git -C spack checkout $SPACK_VERSION
      fi

    - spack/bin/spack gpg trust $GPG_PRIVATE_KEY

    - |
      mkdir -p /stack/extra_repos/

      EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' config.json)

      echo "Repos: ${EXTERNAL_REPOS}"

      for repo in ${EXTERNAL_REPOS}
      do
        repo_branch=$(jq -r ".spack.repos.\"${repo}\".branch" config.json)
        if [ ! -d /stack/extra_repos/${repo} ]; then
          url_branch=$(jq -r ".spack.repos.\"${repo}\".url" config.json)
          git clone -b ${repo_branch} ${url_branch} /stack/extra_repos/$repo
        else
          git -C /stack/extra_repos/$repo fetch
          git -C /stack/extra_repos/$repo checkout ${repo_branch}
        fi
      done

    - mkdir -p /stack/spack/etc
    - cp mirrors.yaml /stack/spack/etc
    - cp repos.yaml /stack/spack/etc
    - cp packages.yaml /stack/spack/etc

    - 'echo "config: { install_tree: { root: /stack/spack/opt/spack } }" > /stack/spack/etc/config.yaml'

    - |
      if [ ! -d /buildcache/build_cache ]; then
        spack/bin/spack gpg publish -d /buildcache
      fi
    - spack/bin/spack buildcache keys --install --trust
    - spack/bin/spack buildcache update-index /buildcache
    - spack/bin/spack buildcache list --allarch




spack:install_compilers:
  stage: compilers
  extends:
    - .spack_cache
    - .parallel_job
  script:
    - COMPILER_LIST_SPECS=$(jq -r '.stack | .core_compiler as $core | .compilers | map("\(.spec)@\(.version) %\($core)")[]' config.json)
    - echo "Compilers to install \n${COMPILER_LIST_SPECS}"

    - spack/bin/spack spec
      --install-status
      --namespaces
      --long
      --types
      ${COMPILER_LIST_SPECS}

    - spack/bin/spack install
      --log-file spack-install.xml
      --log-format junit
      --fail-fast
      --show-log-on-error
      ${COMPILER_LIST_SPECS}

    - spack/bin/spack buildcache create
      --update-index
      --key EDC904DCE3D2E84E
      /buildcache ${COMPILER_LIST_SPECS}

    - |
      echo 'definitions: [ compilers: [' ${COMPILER_LIST_SPECS} ']]' > compiler_definitions.yaml

    - spack/bin/spack compiler find --scope system /usr

    - |
      while read -r compiler
      do
        spack/bin/spack compiler find --scope system $(spack/bin/spack location -i ${compiler})
      done <<< $(echo "${COMPILER_LIST_SPECS}")
  artifacts:
    reports:
      junit: spack-install.xml
  needs:
    - job: spack:setup

spack:concretize:
  stage: concretize
  extends:
    - .spack_cache
    - .parallel_job
  script:
    - spack/bin/spack -e . concretize
  needs:
    - job: spack:install_compilers
  artifacts:
    paths:
      - spack.lock

spack:install:
  stage: install
  extends:
    - .spack_cache
    - .parallel_job
  script:
    - spack/bin/spack -e . install
      --log-file spack-install.xml
      --log-format junit
      --only-concrete
      --fail-fast
      --show-log-on-error
  needs:
    - job: spack:concretize
  artifacts:
    reports:
      junit: spack-install.xml