From a6bb7fc9db30d5e4f1d1f99d6a75fca568251391 Mon Sep 17 00:00:00 2001
From: Nicolas Richart <nicolas.richart@epfl.ch>
Date: Mon, 15 Jul 2024 10:29:43 +0200
Subject: [PATCH] Pulling changes from pinot-noir-gcc to pinot-noir
---
stacks/pinot-noir-gcc/packages_jed.yaml | 5 ++
stacks/pinot-noir/config.yaml | 2 +-
stacks/pinot-noir/definitions_cuda.yaml | 8 ++
stacks/pinot-noir/definitions_empty.yaml | 1 +
stacks/pinot-noir/definitions_helvetios.yaml | 2 +-
stacks/pinot-noir/definitions_izar.yaml | 1 +
stacks/pinot-noir/definitions_kuma_h100.yaml | 8 +-
stacks/pinot-noir/definitions_kuma_l40s.yaml | 1 +
stacks/pinot-noir/modules.yaml | 48 ++++--------
stacks/pinot-noir/modules_empty.yaml | 3 +
stacks/pinot-noir/modules_helvetios.yaml | 1 +
stacks/pinot-noir/modules_izar.yaml | 1 +
stacks/pinot-noir/modules_jed.yaml | 1 +
stacks/pinot-noir/modules_kuma_h100.yaml | 9 +++
stacks/pinot-noir/modules_kuma_l40s.yaml | 1 +
stacks/pinot-noir/packages.yaml | 13 ++++
stacks/pinot-noir/packages_kuma_h100.yaml | 79 +++++++++++++++++++-
stacks/pinot-noir/packages_kuma_l40s.yaml | 79 +++++++++++++++++++-
stacks/pinot-noir/packages_stack.yaml | 6 --
stacks/pinot-noir/spack.yaml | 1 +
20 files changed, 215 insertions(+), 55 deletions(-)
create mode 100644 stacks/pinot-noir/definitions_cuda.yaml
create mode 100644 stacks/pinot-noir/definitions_empty.yaml
mode change 100644 => 120000 stacks/pinot-noir/definitions_helvetios.yaml
create mode 120000 stacks/pinot-noir/definitions_izar.yaml
mode change 100644 => 120000 stacks/pinot-noir/definitions_kuma_h100.yaml
create mode 120000 stacks/pinot-noir/definitions_kuma_l40s.yaml
create mode 100644 stacks/pinot-noir/modules_empty.yaml
create mode 120000 stacks/pinot-noir/modules_helvetios.yaml
create mode 120000 stacks/pinot-noir/modules_izar.yaml
create mode 120000 stacks/pinot-noir/modules_jed.yaml
create mode 100644 stacks/pinot-noir/modules_kuma_h100.yaml
create mode 120000 stacks/pinot-noir/modules_kuma_l40s.yaml
create mode 100644 stacks/pinot-noir/packages.yaml
diff --git a/stacks/pinot-noir-gcc/packages_jed.yaml b/stacks/pinot-noir-gcc/packages_jed.yaml
index 186b82d..822d1e2 100644
--- a/stacks/pinot-noir-gcc/packages_jed.yaml
+++ b/stacks/pinot-noir-gcc/packages_jed.yaml
@@ -23,6 +23,11 @@ packages:
- spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget'
when: '%gcc'
+ kokkos-kernels:
+ require:
+ - spec: '+cuda cuda_arch=90'
+ when: '%gcc'
+
openmpi:
require:
- spec: '+cuda cuda_arch=90 target=icelake'
diff --git a/stacks/pinot-noir/config.yaml b/stacks/pinot-noir/config.yaml
index aff2737..bac71f0 100644
--- a/stacks/pinot-noir/config.yaml
+++ b/stacks/pinot-noir/config.yaml
@@ -3,7 +3,7 @@ config:
install_missing_compilers: false
install_tree:
- padded_length: 242
+ padded_length: False
build_stage:
- $tempdir/$user/spack-stage
diff --git a/stacks/pinot-noir/definitions_cuda.yaml b/stacks/pinot-noir/definitions_cuda.yaml
new file mode 100644
index 0000000..4710276
--- /dev/null
+++ b/stacks/pinot-noir/definitions_cuda.yaml
@@ -0,0 +1,8 @@
+definitions:
+ - cuda_system_codes:
+ - cuda
+ - cudnn
+
+ - cuda_serial_codes:
+ - nccl
+ - nvshmem
diff --git a/stacks/pinot-noir/definitions_empty.yaml b/stacks/pinot-noir/definitions_empty.yaml
new file mode 100644
index 0000000..ca4ba7f
--- /dev/null
+++ b/stacks/pinot-noir/definitions_empty.yaml
@@ -0,0 +1 @@
+definitions: []
diff --git a/stacks/pinot-noir/definitions_helvetios.yaml b/stacks/pinot-noir/definitions_helvetios.yaml
deleted file mode 100644
index ca4ba7f..0000000
--- a/stacks/pinot-noir/definitions_helvetios.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions: []
diff --git a/stacks/pinot-noir/definitions_helvetios.yaml b/stacks/pinot-noir/definitions_helvetios.yaml
new file mode 120000
index 0000000..3552a78
--- /dev/null
+++ b/stacks/pinot-noir/definitions_helvetios.yaml
@@ -0,0 +1 @@
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_izar.yaml b/stacks/pinot-noir/definitions_izar.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir/definitions_izar.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_kuma_h100.yaml b/stacks/pinot-noir/definitions_kuma_h100.yaml
deleted file mode 100644
index 0f156fa..0000000
--- a/stacks/pinot-noir/definitions_kuma_h100.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir/definitions_kuma_h100.yaml b/stacks/pinot-noir/definitions_kuma_h100.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir/definitions_kuma_h100.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_kuma_l40s.yaml b/stacks/pinot-noir/definitions_kuma_l40s.yaml
new file mode 120000
index 0000000..2c65e6a
--- /dev/null
+++ b/stacks/pinot-noir/definitions_kuma_l40s.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/modules.yaml b/stacks/pinot-noir/modules.yaml
index 99f6f3e..38fcc67 100644
--- a/stacks/pinot-noir/modules.yaml
+++ b/stacks/pinot-noir/modules.yaml
@@ -4,13 +4,13 @@ modules:
lmod:
hash_length: 0
core_compilers:
- - gcc@11.4.1
+ - gcc@11.2.1
hierarchy:
- mpi
all:
environment:
set:
- ${PACKAGE}_ROOT: ${PREFIX}
+ "{name}_ROOT": "{prefix}"
suffixes:
'+mpi': mpi
'+openmp': openmp
@@ -25,20 +25,21 @@ modules:
'+ilp64': int64
'+int64': int64
'version_suffix=jl': julia
+ autoload: run
- blacklist_implicits: true
+ hide_implicits: true
- blacklist:
+ exclude:
- lmod
gcc:
environment:
set:
- CC: ${PREFIX}/bin/gcc
- CXX: ${PREFIX}/bin/g++
- F77: ${PREFIX}/bin/gfortran
- FC: ${PREFIX}/bin/gfortran
- F90: ${PREFIX}/bin/gfortran
+ CC: "{prefix}/bin/gcc"
+ CXX: "{prefix}/bin/g++"
+ F77: "{prefix}/bin/gfortran"
+ FC: "{prefix}/bin/gfortran"
+ F90: "{prefix}/bin/gfortran"
cuda@12.4.0:
environment:
append_path:
@@ -47,7 +48,7 @@ modules:
openmpi:
environment:
set:
- SLURM_MPI_TYPE: pmi2
+ SLURM_MPI_TYPE: pmix
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
append_path:
JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
@@ -72,8 +73,6 @@ modules:
I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so
I_MPI_EXTRA_FILESYSTEM: '0'
SLURM_MPI_TYPE: pmi2
- julia:
- autoload: direct
metis:
suffixes:
~real64: sp
@@ -82,26 +81,12 @@ modules:
plumed:
environment:
set:
- PLUMED_KERNEL: ${PREFIX}/lib/libplumedKernel.so
+ PLUMED_KERNEL: "{prefix}/lib/libplumedKernel.so"
unset:
- PLUMED_ROOT
- py-horovod:
- autoload: direct
- py-keras:
- autoload: direct
- py-tensorflow:
- autoload: direct
- py-theano:
- autoload: direct
- py-torch:
- autoload: direct
- py-torchvision:
- autoload: direct
quantum-espresso:
suffixes:
hdf5=parallel: hdf5
- scala:
- autoload: direct
scons:
suffixes:
'^python@:2.99': py2
@@ -116,23 +101,22 @@ modules:
comsol:
environment:
prepend_path:
- MATLABPATH: ${PREFIX}/mli
+ MATLABPATH: "{prefix}/mli"
crystal17:
template: modules/group_restricted.lua
fdtd:
environment:
prepend_path:
- PATH: ${PREFIX}/mpich2/nemesis/bin
- LD_LIBRARY_PATH: ${PREFIX}/mpich2/nemesis/lib
+ PATH: "{prefix}/mpich2/nemesis/bin"
+ LD_LIBRARY_PATH: "{prefix}/mpich2/nemesis/lib"
set:
SLURM_MPI_TYPE: pmi2
gaussian:
template: modules/group_restricted.lua
spark:
- autoload: direct
environment:
prepend_path:
- PATH: /ssoft/spack/scripts/all/spark
+ PATH: "/ssoft/spack/scripts/all/spark"
ucx:
environment:
unset:
diff --git a/stacks/pinot-noir/modules_empty.yaml b/stacks/pinot-noir/modules_empty.yaml
new file mode 100644
index 0000000..21d0544
--- /dev/null
+++ b/stacks/pinot-noir/modules_empty.yaml
@@ -0,0 +1,3 @@
+modules:
+ default:
+ lmod: {}
diff --git a/stacks/pinot-noir/modules_helvetios.yaml b/stacks/pinot-noir/modules_helvetios.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir/modules_helvetios.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/modules_izar.yaml b/stacks/pinot-noir/modules_izar.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir/modules_izar.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/modules_jed.yaml b/stacks/pinot-noir/modules_jed.yaml
new file mode 120000
index 0000000..5be0f31
--- /dev/null
+++ b/stacks/pinot-noir/modules_jed.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/modules_kuma_h100.yaml b/stacks/pinot-noir/modules_kuma_h100.yaml
new file mode 100644
index 0000000..c465907
--- /dev/null
+++ b/stacks/pinot-noir/modules_kuma_h100.yaml
@@ -0,0 +1,9 @@
+modules:
+ default:
+ lmod:
+ openmpi:
+ environment:
+ set:
+ OMPI_MCA_pml: 'ucx'
+ OMPI_MCA_osc: 'ucx'
+ UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'
diff --git a/stacks/pinot-noir/modules_kuma_l40s.yaml b/stacks/pinot-noir/modules_kuma_l40s.yaml
new file mode 120000
index 0000000..7180f05
--- /dev/null
+++ b/stacks/pinot-noir/modules_kuma_l40s.yaml
@@ -0,0 +1 @@
+modules_kuma_h100.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/packages.yaml b/stacks/pinot-noir/packages.yaml
new file mode 100644
index 0000000..01e4aa6
--- /dev/null
+++ b/stacks/pinot-noir/packages.yaml
@@ -0,0 +1,13 @@
+packages:
+ all:
+ providers:
+ mpi: [openmpi, intel-oneapi-mpi]
+ iconv: [libiconv]
+ jpeg: [libjpeg]
+ zlib-api: [zlib-ng]
+ require:
+ - spec: "^libiconv"
+ when: "^iconv"
+
+ zlib-ng:
+ require: ['build_system=autotools']
diff --git a/stacks/pinot-noir/packages_kuma_h100.yaml b/stacks/pinot-noir/packages_kuma_h100.yaml
index 98433e3..281c152 100644
--- a/stacks/pinot-noir/packages_kuma_h100.yaml
+++ b/stacks/pinot-noir/packages_kuma_h100.yaml
@@ -2,10 +2,81 @@ packages:
all:
target: ["zen4"]
require:
- - spec: ^openmpi +cuda
- when: '%gcc ^mpi'
+ - "~rocm"
- - spec: cuda_arch=70
+ - spec: cuda_arch=90 target=zen4
when: '+cuda'
- - '+cuda'
+ - spec: '+cuda'
+ when: '%gcc'
+
+ # ---------------------------------------------------------------------------
+ hypre:
+ require:
+ - spec: '+cuda cuda_arch=90 +unified-memory'
+ when: '%gcc'
+
+ kokkos:
+ require:
+ - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget'
+ when: '%gcc'
+
+ nvshmem:
+ require:
+ - spec: '+cuda cuda_arch=90 +ibgda +nccl +ucx +mpi'
+
+ openmpi:
+ require:
+ - spec: '+cuda cuda_arch=90 target=zen4'
+
+ petsc:
+ require:
+ - spec: '+cuda cuda_arch=90'
+ when: '%gcc'
+
+ py-tensorflow:
+ require:
+ - spec: '+cuda cuda_arch=90 +nccl'
+
+ py-torch:
+ require:
+ - spec: '+cuda cuda_arch=90 +nccl +cudnn'
+
+ quantum-espresso:
+ require:
+ - spec: '~cuda'
+
+ suite-sparse:
+ require:
+ - spec: '+cuda'
+
+ ucx:
+ require:
+ - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=90 ~rocm target=zen4'
+
+ # ---------------------------------------------------------------------------
+ # Externals
+ # ---------------------------------------------------------------------------
+ pmix:
+ buildable: false
+ externals:
+ - spec: 'pmix@5.0.2'
+ prefix: /usr
+
+ rdma-core:
+ buildable: false
+ externals:
+ - spec: 'rdma-core@51.1'
+ prefix: /usr
+
+ slurm:
+ buildable: false
+ externals:
+ - spec: 'slurm@24-05-0-2'
+ prefix: /usr
+
+ xpmem:
+ buildable: false
+ externals:
+ - spec: 'xpmem@2.7.3'
+ prefix: /usr
diff --git a/stacks/pinot-noir/packages_kuma_l40s.yaml b/stacks/pinot-noir/packages_kuma_l40s.yaml
index ad6b8fb..a1b6aef 100644
--- a/stacks/pinot-noir/packages_kuma_l40s.yaml
+++ b/stacks/pinot-noir/packages_kuma_l40s.yaml
@@ -2,10 +2,81 @@ packages:
all:
target: ["zen4"]
require:
- - spec: ^openmpi +cuda
- when: '%gcc ^mpi'
+ - "~rocm"
- - spec: cuda_arch=89
+ - spec: cuda_arch=89 target=zen4
when: '+cuda'
- - '+cuda'
+ - spec: '+cuda'
+ when: '%gcc'
+
+ # ---------------------------------------------------------------------------
+ hypre:
+ require:
+ - spec: '+cuda cuda_arch=89 +unified-memory'
+ when: '%gcc'
+
+ kokkos:
+ require:
+ - spec: '+cuda cuda_arch=89 +cuda_uvm +wrapper ~openmptarget'
+ when: '%gcc'
+
+ nvshmem:
+ require:
+ - spec: '+cuda cuda_arch=89 +ibgda +nccl +ucx +mpi'
+
+ openmpi:
+ require:
+ - spec: '+cuda cuda_arch=89 target=zen4'
+
+ petsc:
+ require:
+ - spec: '+cuda cuda_arch=89'
+ when: '%gcc'
+
+ py-tensorflow:
+ require:
+ - spec: '+cuda cuda_arch=89 +nccl'
+
+ py-torch:
+ require:
+ - spec: '+cuda cuda_arch=89 +nccl +cudnn'
+
+ quantum-espresso:
+ require:
+ - spec: '~cuda'
+
+ suite-sparse:
+ require:
+ - spec: '+cuda'
+
+ ucx:
+ require:
+ - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=89 ~rocm target=zen4'
+
+ # ---------------------------------------------------------------------------
+ # Externals
+ # ---------------------------------------------------------------------------
+ pmix:
+ buildable: false
+ externals:
+ - spec: 'pmix@5.0.2'
+ prefix: /usr
+
+ rdma-core:
+ buildable: false
+ externals:
+ - spec: 'rdma-core@51.1'
+ prefix: /usr
+
+ slurm:
+ buildable: false
+ externals:
+ - spec: 'slurm@24-05-0-2'
+ prefix: /usr
+
+ xpmem:
+ buildable: false
+ externals:
+ - spec: 'xpmem@2.7.3'
+ prefix: /usr
diff --git a/stacks/pinot-noir/packages_stack.yaml b/stacks/pinot-noir/packages_stack.yaml
index d12a3aa..65bebbf 100644
--- a/stacks/pinot-noir/packages_stack.yaml
+++ b/stacks/pinot-noir/packages_stack.yaml
@@ -6,9 +6,6 @@ packages:
jpeg: [libjpeg]
zlib-api: [zlib-ng]
require:
- - spec: "^libiconv"
- when: "^iconv"
-
- spec: '+ipo'
when: '%gcc'
@@ -133,9 +130,6 @@ packages:
require:
- spec: '+rdmacm +rc +dc +ud +cma +verbs'
- zlib-ng:
- require: ['build_system=autotools']
-
# ----------------------------------------------------------------------------
# System dependencies
# ----------------------------------------------------------------------------
diff --git a/stacks/pinot-noir/spack.yaml b/stacks/pinot-noir/spack.yaml
index d80e2cc..4450501 100644
--- a/stacks/pinot-noir/spack.yaml
+++ b/stacks/pinot-noir/spack.yaml
@@ -6,6 +6,7 @@ spack:
- packages_env.yaml
- modules_stack.yaml
+ - modules_env.yaml
- definitions_env.yaml
--
GitLab