From 04309d77fe177bf4dacae45c8079f00669c14a35 Mon Sep 17 00:00:00 2001
From: Nicolas Richart <networms@gmail.ch>
Date: Wed, 10 Jul 2024 08:39:58 +0200
Subject: [PATCH] Adapting information for modules
---
ci/setup_spack.sh | 41 +++++++++------
stacks/pinot-noir-gcc/definitions_cuda.yaml | 7 +++
stacks/pinot-noir-gcc/definitions_empty.yaml | 1 +
.../pinot-noir-gcc/definitions_helvetios.yaml | 2 +-
stacks/pinot-noir-gcc/definitions_izar.yaml | 1 +
.../pinot-noir-gcc/definitions_kuma_h100.yaml | 8 +--
.../pinot-noir-gcc/definitions_kuma_l40s.yaml | 8 +--
stacks/pinot-noir-gcc/modules.yaml | 4 +-
stacks/pinot-noir-gcc/modules_empty.yaml | 3 ++
stacks/pinot-noir-gcc/modules_helvetios.yaml | 1 +
stacks/pinot-noir-gcc/modules_izar.yaml | 1 +
stacks/pinot-noir-gcc/modules_jed.yaml | 1 +
stacks/pinot-noir-gcc/modules_kuma_h100.yaml | 9 ++++
stacks/pinot-noir-gcc/modules_kuma_l40s.yaml | 1 +
stacks/pinot-noir-gcc/packages_stack.yaml | 52 +++----------------
stacks/pinot-noir-gcc/spack.yaml | 2 +-
16 files changed, 64 insertions(+), 78 deletions(-)
create mode 100644 stacks/pinot-noir-gcc/definitions_cuda.yaml
create mode 100644 stacks/pinot-noir-gcc/definitions_empty.yaml
mode change 100644 => 120000 stacks/pinot-noir-gcc/definitions_helvetios.yaml
create mode 120000 stacks/pinot-noir-gcc/definitions_izar.yaml
mode change 100644 => 120000 stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
mode change 100644 => 120000 stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
create mode 100644 stacks/pinot-noir-gcc/modules_empty.yaml
create mode 120000 stacks/pinot-noir-gcc/modules_helvetios.yaml
create mode 120000 stacks/pinot-noir-gcc/modules_izar.yaml
create mode 120000 stacks/pinot-noir-gcc/modules_jed.yaml
create mode 100644 stacks/pinot-noir-gcc/modules_kuma_h100.yaml
create mode 120000 stacks/pinot-noir-gcc/modules_kuma_l40s.yaml
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index 81af918..66c6506 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -32,6 +32,7 @@ do
echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
cp "${STACK_CONFIG_PATH}/${file}.yaml" \
${SPACK_SYSTEM_CONFIG_PATH}
+ cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
fi
if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
@@ -77,6 +78,28 @@ mirrors:
${mirrors}
EOF
+cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
+
+echo "Setting up packages"
+
+echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+export system_compiler=$(jq -Mrc '
+.stack
+ | .system_arch as $arch
+ | .system_compiler
+ | to_entries
+ | .[].value
+ | "\(.compiler)@\(.version) \($arch)"
+' ${STACK_CONFIG})
+
+jq -Mrc '
+.stack.system_packages
+ | map(" \(.):\n require:\n - spec: \"%\(env.system_compiler)\"")
+ | .[]
+' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
+
echo "Setting up extra repos"
EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
@@ -104,6 +127,8 @@ do
fi
done
+cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
+
echo "Setting up buildcache"
spack/bin/spack gpg trust \
@@ -127,19 +152,3 @@ spack/bin/spack buildcache keys \
--trust
#spack/bin/spack buildcache update-index /buildcache
-
-echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-export system_compiler=$(jq -Mrc '
-.stack
- | .system_arch as $arch
- | .system_compiler
- | to_entries
- | .[].value
- | "\(.compiler)@\(.version) \($arch)"
-' ${STACK_CONFIG})
-
-jq -Mrc '
-.stack.system_packages
- | map(" \(.):\n require:\n - spec: \"%\(env.system_compiler)\"")
- | .[]
-' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
diff --git a/stacks/pinot-noir-gcc/definitions_cuda.yaml b/stacks/pinot-noir-gcc/definitions_cuda.yaml
new file mode 100644
index 0000000..0f156fa
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_cuda.yaml
@@ -0,0 +1,7 @@
+definitions:
+ - cuda_system_codes:
+ - cuda
+ - cudnn
+
+ - cuda_serial_codes:
+ - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_empty.yaml b/stacks/pinot-noir-gcc/definitions_empty.yaml
new file mode 100644
index 0000000..ca4ba7f
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_empty.yaml
@@ -0,0 +1 @@
+definitions: []
diff --git a/stacks/pinot-noir-gcc/definitions_helvetios.yaml b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
deleted file mode 100644
index ca4ba7f..0000000
--- a/stacks/pinot-noir-gcc/definitions_helvetios.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions: []
diff --git a/stacks/pinot-noir-gcc/definitions_helvetios.yaml b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
new file mode 120000
index 0000000..3552a78
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
@@ -0,0 +1 @@
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_izar.yaml b/stacks/pinot-noir-gcc/definitions_izar.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_izar.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
deleted file mode 100644
index 0f156fa..0000000
--- a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
deleted file mode 100644
index 0f156fa..0000000
--- a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules.yaml b/stacks/pinot-noir-gcc/modules.yaml
index 99f6f3e..afc56d3 100644
--- a/stacks/pinot-noir-gcc/modules.yaml
+++ b/stacks/pinot-noir-gcc/modules.yaml
@@ -4,7 +4,7 @@ modules:
lmod:
hash_length: 0
core_compilers:
- - gcc@11.4.1
+ - gcc@11.2.1
hierarchy:
- mpi
all:
@@ -47,7 +47,7 @@ modules:
openmpi:
environment:
set:
- SLURM_MPI_TYPE: pmi2
+ SLURM_MPI_TYPE: pmix
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
append_path:
JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
diff --git a/stacks/pinot-noir-gcc/modules_empty.yaml b/stacks/pinot-noir-gcc/modules_empty.yaml
new file mode 100644
index 0000000..21d0544
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_empty.yaml
@@ -0,0 +1,3 @@
+modules:
+ default:
+ lmod: {}
diff --git a/stacks/pinot-noir-gcc/modules_helvetios.yaml b/stacks/pinot-noir-gcc/modules_helvetios.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_helvetios.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_izar.yaml b/stacks/pinot-noir-gcc/modules_izar.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_izar.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_jed.yaml b/stacks/pinot-noir-gcc/modules_jed.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_jed.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_kuma_h100.yaml b/stacks/pinot-noir-gcc/modules_kuma_h100.yaml
new file mode 100644
index 0000000..c465907
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_kuma_h100.yaml
@@ -0,0 +1,9 @@
+modules:
+ default:
+ lmod:
+ openmpi:
+ environment:
+ set:
+ OMPI_MCA_pml: 'ucx'
+ OMPI_MCA_osc: 'ucx'
+ UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'
diff --git a/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml b/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml
new file mode 120000
index 0000000..7180f05
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml
@@ -0,0 +1 @@
+modules_kuma_h100.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/packages_stack.yaml b/stacks/pinot-noir-gcc/packages_stack.yaml
index c9f989d..d12a3aa 100644
--- a/stacks/pinot-noir-gcc/packages_stack.yaml
+++ b/stacks/pinot-noir-gcc/packages_stack.yaml
@@ -9,48 +9,6 @@ packages:
- spec: "^libiconv"
when: "^iconv"
- # - spec: ^intel-oneapi-mpi
- # when: '%oneapi ^mpi'
-
- # - spec: ^openmpi
- # when: '%gcc ^mpi'
-
- # - spec: ^openmpi
- # when: '%aocc ^mpi'
-
- # - spec: ^nvhpc+mpi
- # when: '%nvhpc ^mpi'
-
- # - spec: ^openblas %gcc
- # when: '%gcc ^blas'
-
- # - spec: ^eigen %gcc
- # when: '%gcc ^eigen'
-
- # - spec: ^kokkos %gcc
- # when: '%gcc ^kokkos'
-
- # - spec: ^openblas
- # when: '%gcc ^lapack'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^blas'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^lapack'
-
- # - spec: ^nvhpc+blas
- # when: '%nvhpc ^blas'
-
- # - spec: ^nvhpc+lapack
- # when: '%nvhpc ^lapack'
-
- # - spec: ^openblas
- # when: '%gcc ^py-numpy'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^py-numpy'
-
- spec: '+ipo'
when: '%gcc'
@@ -102,9 +60,12 @@ packages:
prefer: ['~cluster']
kokkos:
- prefer: ['+openmp', '+openmptarget', '+numactl', '+memkind', '+threads',
+ prefer: ['+openmp', '+numactl', '+memkind', '+threads',
'+aggressive_vectorization']
+ kokkos-kernel:
+ prefer: ['+openmp', '+threads']
+
lammps:
require: ['build_type=Release', '+asphere', '+atc', '+body', '+class2',
'+colloid', '+compress', '+coreshell', '+dipole', '+diffraction', '+extra-dump',
@@ -158,6 +119,9 @@ packages:
py-torch:
require: ['+mpi']
+ scotch:
+ prefer: ['~mpi']
+
suite-sparse:
require:
- spec: '+graphblas'
@@ -167,7 +131,7 @@ packages:
ucx:
require:
- - spec: +rdmacm +rc +dc +ud +cma +verbs
+ - spec: '+rdmacm +rc +dc +ud +cma +verbs'
zlib-ng:
require: ['build_system=autotools']
diff --git a/stacks/pinot-noir-gcc/spack.yaml b/stacks/pinot-noir-gcc/spack.yaml
index 19d91c0..e355a2a 100644
--- a/stacks/pinot-noir-gcc/spack.yaml
+++ b/stacks/pinot-noir-gcc/spack.yaml
@@ -5,7 +5,7 @@ spack:
- packages_stack.yaml
- packages_env.yaml
- - modules_stack.yaml
+ - modules_env.yaml
- definitions_env.yaml
--
GitLab