initial commit

.gitlab-ci.yml

+# yaml-language-server: $schema=gitlab-ci
+# yaml-language-server: $format.enable=false
+
+stages:
+  - environment
+  - concretize
+  - install
+  - deploy
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH == "release/*/v*"
+
+variables:
+  BUILDCACHE: /scratch/richart/buildcache
+
+# ------------------------------------------------------------------------------
+.parallel_job:
+  parallel:
+    matrix:
+      - environment: [helvetios]
+        slurm_options: ["-c 36"]
+      # - environment: [jed]
+      # - environment: [izar]
+      #   slurm_options: ['-c 40 --gres gpu:2']
+      #   apptainer_options: ['-nv']
+  tags:
+    - ${environment}
+    - stack
+  variables:
+    COMMAND_OPTIONS_SBATCH: ${slurm_options}
+    APPTAINER_EXEC_OPTIONS: ${apptainer_options} --bind ${BUILDCACHE}:/buildcache:rw --writable-tmpfs
+
+.setup_spack:
+  cache:
+    key: extra-repos-${environment}
+    paths:
+      - /stack/extra_repos
+  before_script:
+    - /opt/spack/bin/spack gpg trust $GPG_PRIVATE_KEY
+
+# ------------------------------------------------------------------------------
+spack:setup:
+  stage:
+    - environment
+  extend:
+    - .parallel_job
+    - .setup_spack
+  script:
+    - >
+      if [ ! -d /buildcache/build_cache ]; then
+        /opt/spack/bin/spack gpg publish -d /buildcache
+      fi
+
+    - mkdir -p /stack/extra_repos
+    - cd /stack/extra_repos
+
+    - git clone https://gitlab.epfl.ch/SCITAS/software-stack/spack-repo-externals.git
+    - git clone https://gitlab.epfl.ch/SCITAS/software-stack/scitas-spack-packages.git
+
+spack:concretize:
+  stage:
+    - concretize
+  extend:
+    - .parallel_job
+    - .setup_spack
+  script:
+    - /opt/spack/bin/spack -e . concretize
+  needs:
+    - job: spack:setup
+  artefacts:
+    paths:
+      - spack.lock
+
+spack:install:
+  stage:
+    - install
+  extend:
+    - .parallel_job
+    - .setup_spack
+  script:
+    - /opt/spack/bin/spack -e . install
+      --log-file spack-install.xml
+      --log-format junit
+      --only-concrete
+      --fail-fast
+      --show-log-on-error
+      --use-cache
+  needs:
+    - job: spack:concretize

dockerfiles/rhel9/Dockerfile

+FROM rhel/ubi9
+
+ARG SPACK_VERSION=v0.21.1
+
+ENV SPACK_DISABLE_LOCAL_CONFIG=true   \
+    SPACK_USER_CACHE_PATH=/tmp/spack-cache
+
+RUN echo "10.95.33.172  foreman1.hpc.epfl.ch" >> /etc/hosts
+RUN curl -sS --insecure \
+    'https://foreman1.hpc.epfl.ch/register?activation_keys=ak-helvetios&organization_id=1&update_packages=false' \
+    -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjo2LCJpYXQiOjE3MDU1NzI0NjYsImp0aSI6IjdkYjE3ODU1NGRhOWVhOTAxYTJjZTY0MmQ5OTFjODAyYmQzZWU1NDI1YmNjYWY4ODY3NjAwNDYwMGE4YmEyNzYiLCJleHAiOjE3MDU1ODY4NjYsInNjb3BlIjoicmVnaXN0cmF0aW9uI2dsb2JhbCByZWdpc3RyYXRpb24jaG9zdCJ9.ilNKpgg__g5RHlDOuwDxtFiTMKLNc-JvPDyWrgy48Pc' \
+    | bash
+
+RUN yum install gcc-g++ gcc-gfortran \
+    patchelf \
+    slurm slurm-libpmi pmix \
+    libibverbs libibverbs-utils \
+    rdma-core \
+    git subversion mercurial
+
+RUN cd /opt && git clone https://github.com/spack/spack.git -b $SPACK_VERSION

mirrors.yaml

+mirrors:
+  scitas-buildcache: file:///buildcache

packages.yaml

+packages:
+  all:
+    providers:
+      mpi: [openmpi]
+      jpeg: [libjpeg]
+
+  # ----------------------------------------------------------------------------
+  # System dependencies
+  # ----------------------------------------------------------------------------
+  rdma-core:
+    version: [44.1]
+    buildable: False
+    externals:
+      - spec: rdma-core@44.1
+        prefix: /usr
+  slurm:
+    version: [23-11-0-1]
+    buildable: False
+    externals:
+      - spec: slurm@23-11-0-1
+        prefix: /usr
+
+  # ----------------------------------------------------------------------------
+  # Requirements
+  # ----------------------------------------------------------------------------
+  ucx:
+    require:
+      - spec: "+rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs"
+      - spec: "+cuda cuda_arch=70 +gdrcopy"
+        when: 'env.get("environment", "") == "izar"'
+
+  openmpi:
+    require:
+      - spec: "fabrics=ofi,ucx,verbs schedulers=slurm ~rsh"
+      - spec: "+gpfs"
+        when: 'env.get("environment", "") == "helvetios"'
+      - spec: "+cuda cuda_arch=70"
+        when: 'env.get("environment", "") == "izar"'
+
+  libfabric:
+    require:
+      - spec: "fabrics=mlx,tcp,udp,verbs,sockets,psm3,shm"
+  # ----------------------------------------------------------------------------
+  # External find
+  # ----------------------------------------------------------------------------
+  binutils:
+    externals:
+      - spec: binutils@2.35.2
+        prefix: /usr
+  coreutils:
+    externals:
+      - spec: coreutils@8.32
+        prefix: /usr
+  findutils:
+    externals:
+      - spec: findutils@4.8.0
+        prefix: /usr
+  gawk:
+    externals:
+      - spec: gawk@5.1.0
+        prefix: /usr
+  openssh:
+    externals:
+      - spec: openssh@8.7p1
+        prefix: /usr
+  openssl:
+    externals:
+      - spec: openssl@3.0.7
+        prefix: /usr
+  tar:
+    externals:
+      - spec: tar@1.34
+        prefix: /usr
+  curl:
+    externals:
+      - spec: curl@7.76.1+gssapi+nghttp2
+        prefix: /usr
+  git:
+    externals:
+      - spec: git@2.39.3~tcltk
+        prefix: /usr
+  gmake:
+    externals:
+      - spec: gmake@4.3
+        prefix: /usr
+  pkgconf:
+    externals:
+      - spec: pkgconf@1.7.3
+        prefix: /usr
+  subversion:
+    externals:
+      - spec: subversion@1.14.1
+        prefix: /usr

repos.yaml

+repos:
+  - /stack/extra_repos/spack-repo-externals
+  - /stack/extra_repos/scitas-spack-packages

spack.yaml

+spack:
+  include:
+    - mirrors.yaml
+    - packages.yaml
+    - repos.yaml
+
+  definitions:
+    - compilers:
+        - gcc
+    - mpi_codes:
+        - osu-micro-benchmarks
+    - mpis:
+        - openmpi
+  specs:
+    - $compilers
+    - matrix:
+        - [$mpis]
+        - [$%compilers]
+    - matrix:
+        - [$mpi_codes]
+        - [$^mpis]
+        - [$%compilers]
+
+  view:
+    default:
+      root: /stack/packages/
+      link: all
+      link_type: copy