Merge branch 'main' of gitlab.epfl.ch:SCITAS/software-stack/scitas-software-stacks

.gitlab-ci.yml

@@ -90,33 +90,17 @@ spack:setup:
     - ci/setup_spack.sh
   needs:
     - job: spack:checkout
-  after_script:
-    - tar czf artifacts_${stack}_${environment}.tgz $stack/environment
-  artifacts:
-    paths:
-      - artifacts_${stack}_${environment}.tgz
 
 spack:install_compilers:
   stage: compilers
   extends:
     - .parallel_job
     - .spack_cache
-  before_script:
-    - if [ -e artifacts_${stack}_${environment}.tgz ];
-      then
-        tar xzf artifacts_${stack}_${environment}.tgz;
-      fi
   script:
     - ci/install_compilers.sh
     - source ci/stack_env.sh
     - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame compilers
-  after_script:
-    - tar czf
-        artifacts_${stack}_${environment}.tgz
-        $stack/spack/var/spack/environments
   artifacts:
-    paths:
-      - artifacts_${stack}_${environment}.tgz
     reports:
       junit: spack-install-*.xml
   needs:
@@ -128,36 +112,23 @@ spack:concretize:
   extends:
     - .parallel_job
     - .spack_cache
-  before_script:
-    - if [ -e artifacts_${stack}_${environment}.tgz ];
-      then
-        tar xzf artifacts_${stack}_${environment}.tgz;
-      fi
   script:
     - source ci/stack_env.sh
-    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame
-    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force
-  after_script:
-    - tar czf
-        artifacts_${stack}_${environment}.tgz
-        $stack/spack/var/spack/environments
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame | tee config.log
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force | tee concretize.log
   artifacts:
     paths:
-      - artifacts_${stack}_${environment}.tgz
+      - config.log
+      - concretize.log
   needs:
     - job: spack:install_compilers
-  timeout: 5h
+  timeout: 24h
 
 spack:install:
   stage: install
   extends:
     - .parallel_job
     - .spack_cache
-  before_script:
-    - if [ -e artifacts_${stack}_${environment}.tgz ];
-      then
-        tar xzf artifacts_${stack}_${environment}.tgz;
-      fi
   script:
     - source ci/stack_env.sh
     - echo "{}" > spack-install-${environment}.xml
@@ -174,18 +145,12 @@ spack:install:
       --update-index
       --key EDC904DCE3D2E84E
       /buildcache
-  after_script:
-    - tar czf
-        artifacts_${stack}_${environment}.tgz
-        $stack/spack/var/spack/environments
   needs:
     - job: spack:concretize
   artifacts:
-    paths:
-      - artifacts_${stack}_${environment}.tgz
     reports:
       junit: spack-install-*.xml
-  timeout: 10h
+  timeout: 72h
 
 spack:mksquashfs:
   stage: deploy

ci/setup_spack.sh

@@ -26,12 +26,13 @@ then
   spack/bin/spack env create ${environment}
 fi
 
-for file in mirrors packages concretizer config spack definitions
+for file in mirrors packages concretizer config spack definitions modules
 do
   if [ -e "${STACK_CONFIG_PATH}/${file}.yaml" ]; then
     echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
     cp "${STACK_CONFIG_PATH}/${file}.yaml" \
       ${SPACK_SYSTEM_CONFIG_PATH}
+    cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
   fi
 
   if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
@@ -77,6 +78,28 @@ mirrors:
 ${mirrors}
 EOF
 
+cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
+
+echo "Setting up packages"
+
+echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+export system_compiler=$(jq -Mrc '
+.stack
+ | .system_arch as $arch
+ | .system_compiler
+ | to_entries
+ | .[].value
+ | "\(.compiler)@\(.version) \($arch)"
+' ${STACK_CONFIG})
+
+jq -Mrc '
+.stack.system_packages
+ | map("  \(.):\n    require:\n    - spec: \"%\(env.system_compiler)\"")
+ | .[]
+' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
+
 echo "Setting up extra repos"
 
 EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
@@ -104,6 +127,8 @@ do
   fi
 done
 
+cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
+
 echo "Setting up buildcache"
 
 spack/bin/spack gpg trust \
@@ -127,19 +152,3 @@ spack/bin/spack buildcache keys \
     --trust
 
 #spack/bin/spack buildcache update-index /buildcache
-
-echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-export system_compiler=$(jq -Mrc '
-.stack
- | .system_arch as $arch
- | .system_compiler
- | to_entries
- | .[].value
- | "\(.compiler)@\(.version) \($arch)"
-' ${STACK_CONFIG})
-
-jq -Mrc '
-.stack.system_packages
- | map("  \(.):\n    require:\n    - spec: \"%\(env.system_compiler)\"")
- | .[]
-' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml

stacks/pinot-noir-gcc/config.yaml

@@ -3,7 +3,7 @@ config:
   install_missing_compilers: false
 
   install_tree:
-    padded_length: 227
+    padded_length: 200
 
   build_stage:
     - $tempdir/$user/spack-stage

stacks/pinot-noir-gcc/definitions_cuda.yaml

+definitions:
+  - cuda_system_codes:
+      - cuda
+      - cudnn
+
+  - cuda_serial_codes:
+      - nccl

stacks/pinot-noir-gcc/definitions_empty.yaml

+definitions: []

stacks/pinot-noir-gcc/definitions_helvetios.yaml

-definitions: []

stacks/pinot-noir-gcc/definitions_helvetios.yaml

+definitions_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_izar.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_kuma_h100.yaml

-definitions:
-  - cuda_system_codes:
-      - cuda
-      - cudnn
-
-  - cuda_serial_codes:
-      - nccl

stacks/pinot-noir-gcc/definitions_kuma_h100.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml

-definitions:
-  - cuda_system_codes:
-      - cuda
-      - cudnn
-
-  - cuda_serial_codes:
-      - nccl

stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules.yaml

@@ -4,7 +4,7 @@ modules:
     lmod:
       hash_length: 0
       core_compilers:
-        - gcc@11.4.1
+        - gcc@11.2.1
       hierarchy:
         - mpi
       all:
@@ -47,7 +47,7 @@ modules:
       openmpi:
         environment:
           set:
-            SLURM_MPI_TYPE: pmi2
+            SLURM_MPI_TYPE: pmix
             OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
           append_path:
             JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"

stacks/pinot-noir-gcc/modules_empty.yaml

+modules:
+  default:
+    lmod: {}

stacks/pinot-noir-gcc/modules_helvetios.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_izar.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_jed.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_kuma_h100.yaml

+modules:
+  default:
+    lmod:
+      openmpi:
+        environment:
+          set:
+            OMPI_MCA_pml: 'ucx'
+            OMPI_MCA_osc: 'ucx'
+            UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'

stacks/pinot-noir-gcc/modules_kuma_l40s.yaml

+modules_kuma_h100.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/packages_stack.yaml

@@ -9,48 +9,6 @@ packages:
       - spec: "^libiconv"
         when: "^iconv"
 
-      # - spec: ^intel-oneapi-mpi
-      #   when: '%oneapi ^mpi'
-
-      # - spec: ^openmpi
-      #   when: '%gcc ^mpi'
-
-      # - spec: ^openmpi
-      #   when: '%aocc ^mpi'
-
-      # - spec: ^nvhpc+mpi
-      #   when: '%nvhpc ^mpi'
-
-      # - spec: ^openblas %gcc
-      #   when: '%gcc ^blas'
-
-      # - spec: ^eigen %gcc
-      #   when: '%gcc ^eigen'
-
-      # - spec: ^kokkos %gcc
-      #   when: '%gcc ^kokkos'
-
-      # - spec: ^openblas
-      #   when: '%gcc ^lapack'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^blas'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^lapack'
-
-      # - spec: ^nvhpc+blas
-      #   when: '%nvhpc ^blas'
-
-      # - spec: ^nvhpc+lapack
-      #   when: '%nvhpc ^lapack'
-
-      # - spec: ^openblas
-      #   when: '%gcc ^py-numpy'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^py-numpy'
-
       - spec: '+ipo'
         when: '%gcc'
 
@@ -102,9 +60,12 @@ packages:
     prefer: ['~cluster']
 
   kokkos:
-    prefer: ['+openmp', '+openmptarget', '+numactl', '+memkind', '+threads',
+    prefer: ['+openmp', '+numactl', '+memkind', '+threads',
     '+aggressive_vectorization']
 
+  kokkos-kernel:
+    prefer: ['+openmp', '+threads']
+
   lammps:
     require: ['build_type=Release', '+asphere', '+atc', '+body', '+class2',
     '+colloid', '+compress', '+coreshell', '+dipole', '+diffraction', '+extra-dump',
@@ -158,6 +119,9 @@ packages:
   py-torch:
     require: ['+mpi']
 
+  scotch:
+    prefer: ['~mpi']
+
   suite-sparse:
     require:
     - spec: '+graphblas'
@@ -167,7 +131,7 @@ packages:
 
   ucx:
     require:
-    - spec: +rdmacm +rc +dc +ud +cma +verbs
+    - spec: '+rdmacm +rc +dc +ud +cma +verbs'
 
   zlib-ng:
     require: ['build_system=autotools']