Adapting information for modules

ci/setup_spack.sh

@@ -32,6 +32,7 @@ do
     echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
     cp "${STACK_CONFIG_PATH}/${file}.yaml" \
       ${SPACK_SYSTEM_CONFIG_PATH}
+    cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
   fi
 
   if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
@@ -77,6 +78,28 @@ mirrors:
 ${mirrors}
 EOF
 
+cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
+
+echo "Setting up packages"
+
+echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+export system_compiler=$(jq -Mrc '
+.stack
+ | .system_arch as $arch
+ | .system_compiler
+ | to_entries
+ | .[].value
+ | "\(.compiler)@\(.version) \($arch)"
+' ${STACK_CONFIG})
+
+jq -Mrc '
+.stack.system_packages
+ | map("  \(.):\n    require:\n    - spec: \"%\(env.system_compiler)\"")
+ | .[]
+' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
+
 echo "Setting up extra repos"
 
 EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
@@ -104,6 +127,8 @@ do
   fi
 done
 
+cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
+
 echo "Setting up buildcache"
 
 spack/bin/spack gpg trust \
@@ -127,19 +152,3 @@ spack/bin/spack buildcache keys \
     --trust
 
 #spack/bin/spack buildcache update-index /buildcache
-
-echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-export system_compiler=$(jq -Mrc '
-.stack
- | .system_arch as $arch
- | .system_compiler
- | to_entries
- | .[].value
- | "\(.compiler)@\(.version) \($arch)"
-' ${STACK_CONFIG})
-
-jq -Mrc '
-.stack.system_packages
- | map("  \(.):\n    require:\n    - spec: \"%\(env.system_compiler)\"")
- | .[]
-' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml

stacks/pinot-noir-gcc/definitions_cuda.yaml

+definitions:
+  - cuda_system_codes:
+      - cuda
+      - cudnn
+
+  - cuda_serial_codes:
+      - nccl

stacks/pinot-noir-gcc/definitions_empty.yaml

+definitions: []

stacks/pinot-noir-gcc/definitions_helvetios.yaml

-definitions: []

stacks/pinot-noir-gcc/definitions_helvetios.yaml

+definitions_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_izar.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_kuma_h100.yaml

-definitions:
-  - cuda_system_codes:
-      - cuda
-      - cudnn
-
-  - cuda_serial_codes:
-      - nccl

stacks/pinot-noir-gcc/definitions_kuma_h100.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml

-definitions:
-  - cuda_system_codes:
-      - cuda
-      - cudnn
-
-  - cuda_serial_codes:
-      - nccl

stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules.yaml

@@ -4,7 +4,7 @@ modules:
     lmod:
       hash_length: 0
       core_compilers:
-        - gcc@11.4.1
+        - gcc@11.2.1
       hierarchy:
         - mpi
       all:
@@ -47,7 +47,7 @@ modules:
       openmpi:
         environment:
           set:
-            SLURM_MPI_TYPE: pmi2
+            SLURM_MPI_TYPE: pmix
             OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
           append_path:
             JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"

stacks/pinot-noir-gcc/modules_empty.yaml

+modules:
+  default:
+    lmod: {}

stacks/pinot-noir-gcc/modules_helvetios.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_izar.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_jed.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/modules_kuma_h100.yaml

+modules:
+  default:
+    lmod:
+      openmpi:
+        environment:
+          set:
+            OMPI_MCA_pml: 'ucx'
+            OMPI_MCA_osc: 'ucx'
+            UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'

stacks/pinot-noir-gcc/modules_kuma_l40s.yaml

+modules_kuma_h100.yaml
\ No newline at end of file

stacks/pinot-noir-gcc/packages_stack.yaml

@@ -9,48 +9,6 @@ packages:
       - spec: "^libiconv"
         when: "^iconv"
 
-      # - spec: ^intel-oneapi-mpi
-      #   when: '%oneapi ^mpi'
-
-      # - spec: ^openmpi
-      #   when: '%gcc ^mpi'
-
-      # - spec: ^openmpi
-      #   when: '%aocc ^mpi'
-
-      # - spec: ^nvhpc+mpi
-      #   when: '%nvhpc ^mpi'
-
-      # - spec: ^openblas %gcc
-      #   when: '%gcc ^blas'
-
-      # - spec: ^eigen %gcc
-      #   when: '%gcc ^eigen'
-
-      # - spec: ^kokkos %gcc
-      #   when: '%gcc ^kokkos'
-
-      # - spec: ^openblas
-      #   when: '%gcc ^lapack'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^blas'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^lapack'
-
-      # - spec: ^nvhpc+blas
-      #   when: '%nvhpc ^blas'
-
-      # - spec: ^nvhpc+lapack
-      #   when: '%nvhpc ^lapack'
-
-      # - spec: ^openblas
-      #   when: '%gcc ^py-numpy'
-
-      # - spec: ^intel-oneapi-mkl
-      #   when: '%oneapi ^py-numpy'
-
       - spec: '+ipo'
         when: '%gcc'
 
@@ -102,9 +60,12 @@ packages:
     prefer: ['~cluster']
 
   kokkos:
-    prefer: ['+openmp', '+openmptarget', '+numactl', '+memkind', '+threads',
+    prefer: ['+openmp', '+numactl', '+memkind', '+threads',
     '+aggressive_vectorization']
 
+  kokkos-kernel:
+    prefer: ['+openmp', '+threads']
+
   lammps:
     require: ['build_type=Release', '+asphere', '+atc', '+body', '+class2',
     '+colloid', '+compress', '+coreshell', '+dipole', '+diffraction', '+extra-dump',
@@ -158,6 +119,9 @@ packages:
   py-torch:
     require: ['+mpi']
 
+  scotch:
+    prefer: ['~mpi']
+
   suite-sparse:
     require:
     - spec: '+graphblas'
@@ -167,7 +131,7 @@ packages:
 
   ucx:
     require:
-    - spec: +rdmacm +rc +dc +ud +cma +verbs
+    - spec: '+rdmacm +rc +dc +ud +cma +verbs'
 
   zlib-ng:
     require: ['build_system=autotools']

stacks/pinot-noir-gcc/spack.yaml

@@ -5,7 +5,7 @@ spack:
       - packages_stack.yaml
       - packages_env.yaml
 
-      - modules_stack.yaml
+      - modules_env.yaml
 
       - definitions_env.yaml