Pulling changes from pinot-noir-gcc to pinot-noir

stacks/pinot-noir-gcc/packages_jed.yaml

@@ -23,6 +23,11 @@ packages:
     - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget'
       when: '%gcc'
 
+  kokkos-kernels:
+    require:
+    - spec: '+cuda cuda_arch=90'
+      when: '%gcc'
+
   openmpi:
     require:
       - spec: '+cuda cuda_arch=90 target=icelake'

stacks/pinot-noir/config.yaml

@@ -3,7 +3,7 @@ config:
   install_missing_compilers: false
 
   install_tree:
-    padded_length: 242
+    padded_length: False
 
   build_stage:
     - $tempdir/$user/spack-stage

stacks/pinot-noir/definitions_cuda.yaml

+definitions:
+  - cuda_system_codes:
+      - cuda
+      - cudnn
+
+  - cuda_serial_codes:
+      - nccl
+      - nvshmem

stacks/pinot-noir/definitions_empty.yaml

+definitions: []

stacks/pinot-noir/definitions_helvetios.yaml

-definitions: []

stacks/pinot-noir/definitions_helvetios.yaml

+definitions_empty.yaml
\ No newline at end of file

stacks/pinot-noir/definitions_izar.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir/definitions_kuma_h100.yaml

-definitions:
-  - cuda_system_codes:
-      - cuda
-      - cudnn
-
-  - cuda_serial_codes:
-      - nccl

stacks/pinot-noir/definitions_kuma_h100.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir/definitions_kuma_l40s.yaml

+definitions_cuda.yaml
\ No newline at end of file

stacks/pinot-noir/modules.yaml

@@ -4,13 +4,13 @@ modules:
     lmod:
       hash_length: 0
       core_compilers:
-        - gcc@11.4.1
+        - gcc@11.2.1
       hierarchy:
         - mpi
       all:
         environment:
           set:
-            ${PACKAGE}_ROOT: ${PREFIX}
+            "{name}_ROOT": "{prefix}"
         suffixes:
           '+mpi': mpi
           '+openmp': openmp
@@ -25,20 +25,21 @@ modules:
           '+ilp64': int64
           '+int64': int64
           'version_suffix=jl': julia
+        autoload: run
 
-      blacklist_implicits: true
+      hide_implicits: true
 
-      blacklist:
+      exclude:
         - lmod
 
       gcc:
         environment:
           set:
-            CC: ${PREFIX}/bin/gcc
-            CXX: ${PREFIX}/bin/g++
-            F77: ${PREFIX}/bin/gfortran
-            FC: ${PREFIX}/bin/gfortran
-            F90: ${PREFIX}/bin/gfortran
+            CC: "{prefix}/bin/gcc"
+            CXX: "{prefix}/bin/g++"
+            F77: "{prefix}/bin/gfortran"
+            FC: "{prefix}/bin/gfortran"
+            F90: "{prefix}/bin/gfortran"
       cuda@12.4.0:
         environment:
           append_path:
@@ -47,7 +48,7 @@ modules:
       openmpi:
         environment:
           set:
-            SLURM_MPI_TYPE: pmi2
+            SLURM_MPI_TYPE: pmix
             OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
           append_path:
             JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
@@ -72,8 +73,6 @@ modules:
             I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so
             I_MPI_EXTRA_FILESYSTEM: '0'
             SLURM_MPI_TYPE: pmi2
-      julia:
-        autoload: direct
       metis:
         suffixes:
           ~real64: sp
@@ -82,26 +81,12 @@ modules:
       plumed:
         environment:
           set:
-            PLUMED_KERNEL: ${PREFIX}/lib/libplumedKernel.so
+            PLUMED_KERNEL: "{prefix}/lib/libplumedKernel.so"
           unset:
           - PLUMED_ROOT
-      py-horovod:
-        autoload: direct
-      py-keras:
-        autoload: direct
-      py-tensorflow:
-        autoload: direct
-      py-theano:
-        autoload: direct
-      py-torch:
-        autoload: direct
-      py-torchvision:
-        autoload: direct
       quantum-espresso:
         suffixes:
           hdf5=parallel: hdf5
-      scala:
-        autoload: direct
       scons:
         suffixes:
           '^python@:2.99': py2
@@ -116,23 +101,22 @@ modules:
       comsol:
         environment:
           prepend_path:
-            MATLABPATH: ${PREFIX}/mli
+            MATLABPATH: "{prefix}/mli"
       crystal17:
         template: modules/group_restricted.lua
       fdtd:
         environment:
           prepend_path:
-            PATH: ${PREFIX}/mpich2/nemesis/bin
-            LD_LIBRARY_PATH: ${PREFIX}/mpich2/nemesis/lib
+            PATH: "{prefix}/mpich2/nemesis/bin"
+            LD_LIBRARY_PATH: "{prefix}/mpich2/nemesis/lib"
           set:
             SLURM_MPI_TYPE: pmi2
       gaussian:
         template: modules/group_restricted.lua
       spark:
-        autoload: direct
         environment:
           prepend_path:
-            PATH: /ssoft/spack/scripts/all/spark
+            PATH: "/ssoft/spack/scripts/all/spark"
       ucx:
         environment:
           unset:

stacks/pinot-noir/modules_empty.yaml

+modules:
+  default:
+    lmod: {}

stacks/pinot-noir/modules_helvetios.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir/modules_izar.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir/modules_jed.yaml

+modules_empty.yaml
\ No newline at end of file

stacks/pinot-noir/modules_kuma_h100.yaml

+modules:
+  default:
+    lmod:
+      openmpi:
+        environment:
+          set:
+            OMPI_MCA_pml: 'ucx'
+            OMPI_MCA_osc: 'ucx'
+            UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'

stacks/pinot-noir/modules_kuma_l40s.yaml

+modules_kuma_h100.yaml
\ No newline at end of file

stacks/pinot-noir/packages.yaml

+packages:
+  all:
+    providers:
+      mpi: [openmpi, intel-oneapi-mpi]
+      iconv: [libiconv]
+      jpeg: [libjpeg]
+      zlib-api: [zlib-ng]
+    require:
+      - spec: "^libiconv"
+        when: "^iconv"
+
+  zlib-ng:
+    require: ['build_system=autotools']

stacks/pinot-noir/packages_kuma_h100.yaml

@@ -2,10 +2,81 @@ packages:
   all:
     target: ["zen4"]
     require:
-      - spec: ^openmpi +cuda
-        when: '%gcc ^mpi'
+      - "~rocm"
 
-      - spec: cuda_arch=70
+      - spec: cuda_arch=90 target=zen4
         when: '+cuda'
 
-      - '+cuda'
+      - spec: '+cuda'
+        when: '%gcc'
+
+  # ---------------------------------------------------------------------------
+  hypre:
+    require:
+    - spec: '+cuda cuda_arch=90 +unified-memory'
+      when: '%gcc'
+
+  kokkos:
+    require:
+    - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget'
+      when: '%gcc'
+
+  nvshmem:
+    require:
+    - spec: '+cuda cuda_arch=90 +ibgda +nccl +ucx +mpi'
+
+  openmpi:
+    require:
+      - spec: '+cuda cuda_arch=90 target=zen4'
+
+  petsc:
+    require:
+    - spec: '+cuda cuda_arch=90'
+      when: '%gcc'
+
+  py-tensorflow:
+    require:
+    - spec: '+cuda cuda_arch=90 +nccl'
+
+  py-torch:
+    require:
+    - spec: '+cuda cuda_arch=90 +nccl +cudnn'
+
+  quantum-espresso:
+    require:
+    - spec: '~cuda'
+
+  suite-sparse:
+    require:
+    - spec: '+cuda'
+
+  ucx:
+    require:
+      - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=90 ~rocm target=zen4'
+
+  # ---------------------------------------------------------------------------
+  # Externals
+  # ---------------------------------------------------------------------------
+  pmix:
+    buildable: false
+    externals:
+    - spec: 'pmix@5.0.2'
+      prefix: /usr
+
+  rdma-core:
+    buildable: false
+    externals:
+    - spec: 'rdma-core@51.1'
+      prefix: /usr
+
+  slurm:
+    buildable: false
+    externals:
+    - spec: 'slurm@24-05-0-2'
+      prefix: /usr
+
+  xpmem:
+    buildable: false
+    externals:
+    - spec: 'xpmem@2.7.3'
+      prefix: /usr

stacks/pinot-noir/packages_kuma_l40s.yaml

@@ -2,10 +2,81 @@ packages:
   all:
     target: ["zen4"]
     require:
-      - spec: ^openmpi +cuda
-        when: '%gcc ^mpi'
+      - "~rocm"
 
-      - spec: cuda_arch=89
+      - spec: cuda_arch=89 target=zen4
         when: '+cuda'
 
-      - '+cuda'
+      - spec: '+cuda'
+        when: '%gcc'
+
+  # ---------------------------------------------------------------------------
+  hypre:
+    require:
+    - spec: '+cuda cuda_arch=89 +unified-memory'
+      when: '%gcc'
+
+  kokkos:
+    require:
+    - spec: '+cuda cuda_arch=89 +cuda_uvm +wrapper ~openmptarget'
+      when: '%gcc'
+
+  nvshmem:
+    require:
+    - spec: '+cuda cuda_arch=89 +ibgda +nccl +ucx +mpi'
+
+  openmpi:
+    require:
+      - spec: '+cuda cuda_arch=89 target=zen4'
+
+  petsc:
+    require:
+    - spec: '+cuda cuda_arch=89'
+      when: '%gcc'
+
+  py-tensorflow:
+    require:
+    - spec: '+cuda cuda_arch=89 +nccl'
+
+  py-torch:
+    require:
+    - spec: '+cuda cuda_arch=89 +nccl +cudnn'
+
+  quantum-espresso:
+    require:
+    - spec: '~cuda'
+
+  suite-sparse:
+    require:
+    - spec: '+cuda'
+
+  ucx:
+    require:
+      - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=89 ~rocm target=zen4'
+
+  # ---------------------------------------------------------------------------
+  # Externals
+  # ---------------------------------------------------------------------------
+  pmix:
+    buildable: false
+    externals:
+    - spec: 'pmix@5.0.2'
+      prefix: /usr
+
+  rdma-core:
+    buildable: false
+    externals:
+    - spec: 'rdma-core@51.1'
+      prefix: /usr
+
+  slurm:
+    buildable: false
+    externals:
+    - spec: 'slurm@24-05-0-2'
+      prefix: /usr
+
+  xpmem:
+    buildable: false
+    externals:
+    - spec: 'xpmem@2.7.3'
+      prefix: /usr