diff --git a/stacks/pinot-noir-gcc/packages_jed.yaml b/stacks/pinot-noir-gcc/packages_jed.yaml index 186b82d655ded03701bd8fff6750cd56779a51dd..822d1e2f6b56828d7269ba821ecb27be99c9d6d2 100644 --- a/stacks/pinot-noir-gcc/packages_jed.yaml +++ b/stacks/pinot-noir-gcc/packages_jed.yaml @@ -23,6 +23,11 @@ packages: - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget' when: '%gcc' + kokkos-kernels: + require: + - spec: '+cuda cuda_arch=90' + when: '%gcc' + openmpi: require: - spec: '+cuda cuda_arch=90 target=icelake' diff --git a/stacks/pinot-noir/config.yaml b/stacks/pinot-noir/config.yaml index aff2737c506bb6ddb8fdbe7e8c81676ab70d5ec6..bac71f0dae6d0417c501344add77df7803c2dde1 100644 --- a/stacks/pinot-noir/config.yaml +++ b/stacks/pinot-noir/config.yaml @@ -3,7 +3,7 @@ config: install_missing_compilers: false install_tree: - padded_length: 242 + padded_length: False build_stage: - $tempdir/$user/spack-stage diff --git a/stacks/pinot-noir/definitions_cuda.yaml b/stacks/pinot-noir/definitions_cuda.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4710276c2be849f7ba905a7c66ce192ee3a206eb --- /dev/null +++ b/stacks/pinot-noir/definitions_cuda.yaml @@ -0,0 +1,8 @@ +definitions: + - cuda_system_codes: + - cuda + - cudnn + + - cuda_serial_codes: + - nccl + - nvshmem diff --git a/stacks/pinot-noir/definitions_empty.yaml b/stacks/pinot-noir/definitions_empty.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca4ba7fe51ad3279cebc61ff97ee72782db03b81 --- /dev/null +++ b/stacks/pinot-noir/definitions_empty.yaml @@ -0,0 +1 @@ +definitions: [] diff --git a/stacks/pinot-noir/definitions_helvetios.yaml b/stacks/pinot-noir/definitions_helvetios.yaml deleted file mode 100644 index ca4ba7fe51ad3279cebc61ff97ee72782db03b81..0000000000000000000000000000000000000000 --- a/stacks/pinot-noir/definitions_helvetios.yaml +++ /dev/null @@ -1 +0,0 @@ -definitions: [] diff --git a/stacks/pinot-noir/definitions_helvetios.yaml b/stacks/pinot-noir/definitions_helvetios.yaml new file mode 120000 index 0000000000000000000000000000000000000000..3552a78f67656b1b6647cced51cbfb18b5882f10 --- /dev/null +++ b/stacks/pinot-noir/definitions_helvetios.yaml @@ -0,0 +1 @@ +definitions_empty.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/definitions_izar.yaml b/stacks/pinot-noir/definitions_izar.yaml new file mode 120000 index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f --- /dev/null +++ b/stacks/pinot-noir/definitions_izar.yaml @@ -0,0 +1 @@ +definitions_cuda.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/definitions_kuma_h100.yaml b/stacks/pinot-noir/definitions_kuma_h100.yaml deleted file mode 100644 index 0f156fa830b9567b3e7839218a0a88c16076c6ca..0000000000000000000000000000000000000000 --- a/stacks/pinot-noir/definitions_kuma_h100.yaml +++ /dev/null @@ -1,7 +0,0 @@ -definitions: - - cuda_system_codes: - - cuda - - cudnn - - - cuda_serial_codes: - - nccl diff --git a/stacks/pinot-noir/definitions_kuma_h100.yaml b/stacks/pinot-noir/definitions_kuma_h100.yaml new file mode 120000 index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f --- /dev/null +++ b/stacks/pinot-noir/definitions_kuma_h100.yaml @@ -0,0 +1 @@ +definitions_cuda.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/definitions_kuma_l40s.yaml b/stacks/pinot-noir/definitions_kuma_l40s.yaml new file mode 120000 index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f --- /dev/null +++ b/stacks/pinot-noir/definitions_kuma_l40s.yaml @@ -0,0 +1 @@ +definitions_cuda.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/modules.yaml b/stacks/pinot-noir/modules.yaml index 99f6f3e104d547f086fd83a4e77d1b4f3be316a0..38fcc67d3399d00e3662efcae31e385da1fb39ad 100644 --- a/stacks/pinot-noir/modules.yaml +++ b/stacks/pinot-noir/modules.yaml @@ -4,13 +4,13 @@ modules: lmod: hash_length: 0 core_compilers: - - gcc@11.4.1 + - gcc@11.2.1 hierarchy: - mpi all: environment: set: - ${PACKAGE}_ROOT: ${PREFIX} + "{name}_ROOT": "{prefix}" suffixes: '+mpi': mpi '+openmp': openmp @@ -25,20 +25,21 @@ modules: '+ilp64': int64 '+int64': int64 'version_suffix=jl': julia + autoload: run - blacklist_implicits: true + hide_implicits: true - blacklist: + exclude: - lmod gcc: environment: set: - CC: ${PREFIX}/bin/gcc - CXX: ${PREFIX}/bin/g++ - F77: ${PREFIX}/bin/gfortran - FC: ${PREFIX}/bin/gfortran - F90: ${PREFIX}/bin/gfortran + CC: "{prefix}/bin/gcc" + CXX: "{prefix}/bin/g++" + F77: "{prefix}/bin/gfortran" + FC: "{prefix}/bin/gfortran" + F90: "{prefix}/bin/gfortran" cuda@12.4.0: environment: append_path: @@ -47,7 +48,7 @@ modules: openmpi: environment: set: - SLURM_MPI_TYPE: pmi2 + SLURM_MPI_TYPE: pmix OMPI_MCA_btl_openib_warn_default_gid_prefix: '0' append_path: JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi" @@ -72,8 +73,6 @@ modules: I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so I_MPI_EXTRA_FILESYSTEM: '0' SLURM_MPI_TYPE: pmi2 - julia: - autoload: direct metis: suffixes: ~real64: sp @@ -82,26 +81,12 @@ modules: plumed: environment: set: - PLUMED_KERNEL: ${PREFIX}/lib/libplumedKernel.so + PLUMED_KERNEL: "{prefix}/lib/libplumedKernel.so" unset: - PLUMED_ROOT - py-horovod: - autoload: direct - py-keras: - autoload: direct - py-tensorflow: - autoload: direct - py-theano: - autoload: direct - py-torch: - autoload: direct - py-torchvision: - autoload: direct quantum-espresso: suffixes: hdf5=parallel: hdf5 - scala: - autoload: direct scons: suffixes: '^python@:2.99': py2 @@ -116,23 +101,22 @@ modules: comsol: environment: prepend_path: - MATLABPATH: ${PREFIX}/mli + MATLABPATH: "{prefix}/mli" crystal17: template: modules/group_restricted.lua fdtd: environment: prepend_path: - PATH: ${PREFIX}/mpich2/nemesis/bin - LD_LIBRARY_PATH: ${PREFIX}/mpich2/nemesis/lib + PATH: "{prefix}/mpich2/nemesis/bin" + LD_LIBRARY_PATH: "{prefix}/mpich2/nemesis/lib" set: SLURM_MPI_TYPE: pmi2 gaussian: template: modules/group_restricted.lua spark: - autoload: direct environment: prepend_path: - PATH: /ssoft/spack/scripts/all/spark + PATH: "/ssoft/spack/scripts/all/spark" ucx: environment: unset: diff --git a/stacks/pinot-noir/modules_empty.yaml b/stacks/pinot-noir/modules_empty.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d0544839675931122d4455d2fe32408b425fe5 --- /dev/null +++ b/stacks/pinot-noir/modules_empty.yaml @@ -0,0 +1,3 @@ +modules: + default: + lmod: {} diff --git a/stacks/pinot-noir/modules_helvetios.yaml b/stacks/pinot-noir/modules_helvetios.yaml new file mode 120000 index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4 --- /dev/null +++ b/stacks/pinot-noir/modules_helvetios.yaml @@ -0,0 +1 @@ +modules_empty.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/modules_izar.yaml b/stacks/pinot-noir/modules_izar.yaml new file mode 120000 index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4 --- /dev/null +++ b/stacks/pinot-noir/modules_izar.yaml @@ -0,0 +1 @@ +modules_empty.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/modules_jed.yaml b/stacks/pinot-noir/modules_jed.yaml new file mode 120000 index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4 --- /dev/null +++ b/stacks/pinot-noir/modules_jed.yaml @@ -0,0 +1 @@ +modules_empty.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/modules_kuma_h100.yaml b/stacks/pinot-noir/modules_kuma_h100.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4659078af130b7c2532939538c205d6d12edde9 --- /dev/null +++ b/stacks/pinot-noir/modules_kuma_h100.yaml @@ -0,0 +1,9 @@ +modules: + default: + lmod: + openmpi: + environment: + set: + OMPI_MCA_pml: 'ucx' + OMPI_MCA_osc: 'ucx' + UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1' diff --git a/stacks/pinot-noir/modules_kuma_l40s.yaml b/stacks/pinot-noir/modules_kuma_l40s.yaml new file mode 120000 index 0000000000000000000000000000000000000000..7180f05ea42a27f7e59f063ecb69b4d815c4030b --- /dev/null +++ b/stacks/pinot-noir/modules_kuma_l40s.yaml @@ -0,0 +1 @@ +modules_kuma_h100.yaml \ No newline at end of file diff --git a/stacks/pinot-noir/packages.yaml b/stacks/pinot-noir/packages.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01e4aa6bffd49e9caf3c243de15edba0287bd56f --- /dev/null +++ b/stacks/pinot-noir/packages.yaml @@ -0,0 +1,13 @@ +packages: + all: + providers: + mpi: [openmpi, intel-oneapi-mpi] + iconv: [libiconv] + jpeg: [libjpeg] + zlib-api: [zlib-ng] + require: + - spec: "^libiconv" + when: "^iconv" + + zlib-ng: + require: ['build_system=autotools'] diff --git a/stacks/pinot-noir/packages_kuma_h100.yaml b/stacks/pinot-noir/packages_kuma_h100.yaml index 98433e3b2c1f32a96105e08e00b299ac1312fafe..281c15220290598dbe1e9d38ffc6a9268089bce4 100644 --- a/stacks/pinot-noir/packages_kuma_h100.yaml +++ b/stacks/pinot-noir/packages_kuma_h100.yaml @@ -2,10 +2,81 @@ packages: all: target: ["zen4"] require: - - spec: ^openmpi +cuda - when: '%gcc ^mpi' + - "~rocm" - - spec: cuda_arch=70 + - spec: cuda_arch=90 target=zen4 when: '+cuda' - - '+cuda' + - spec: '+cuda' + when: '%gcc' + + # --------------------------------------------------------------------------- + hypre: + require: + - spec: '+cuda cuda_arch=90 +unified-memory' + when: '%gcc' + + kokkos: + require: + - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget' + when: '%gcc' + + nvshmem: + require: + - spec: '+cuda cuda_arch=90 +ibgda +nccl +ucx +mpi' + + openmpi: + require: + - spec: '+cuda cuda_arch=90 target=zen4' + + petsc: + require: + - spec: '+cuda cuda_arch=90' + when: '%gcc' + + py-tensorflow: + require: + - spec: '+cuda cuda_arch=90 +nccl' + + py-torch: + require: + - spec: '+cuda cuda_arch=90 +nccl +cudnn' + + quantum-espresso: + require: + - spec: '~cuda' + + suite-sparse: + require: + - spec: '+cuda' + + ucx: + require: + - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=90 ~rocm target=zen4' + + # --------------------------------------------------------------------------- + # Externals + # --------------------------------------------------------------------------- + pmix: + buildable: false + externals: + - spec: 'pmix@5.0.2' + prefix: /usr + + rdma-core: + buildable: false + externals: + - spec: 'rdma-core@51.1' + prefix: /usr + + slurm: + buildable: false + externals: + - spec: 'slurm@24-05-0-2' + prefix: /usr + + xpmem: + buildable: false + externals: + - spec: 'xpmem@2.7.3' + prefix: /usr diff --git a/stacks/pinot-noir/packages_kuma_l40s.yaml b/stacks/pinot-noir/packages_kuma_l40s.yaml index ad6b8fb44336cd6730bb956c5d8489ff7f0acf0b..a1b6aef1ba3ec155b9dedcdc3825c7206f3222f0 100644 --- a/stacks/pinot-noir/packages_kuma_l40s.yaml +++ b/stacks/pinot-noir/packages_kuma_l40s.yaml @@ -2,10 +2,81 @@ packages: all: target: ["zen4"] require: - - spec: ^openmpi +cuda - when: '%gcc ^mpi' + - "~rocm" - - spec: cuda_arch=89 + - spec: cuda_arch=89 target=zen4 when: '+cuda' - - '+cuda' + - spec: '+cuda' + when: '%gcc' + + # --------------------------------------------------------------------------- + hypre: + require: + - spec: '+cuda cuda_arch=89 +unified-memory' + when: '%gcc' + + kokkos: + require: + - spec: '+cuda cuda_arch=89 +cuda_uvm +wrapper ~openmptarget' + when: '%gcc' + + nvshmem: + require: + - spec: '+cuda cuda_arch=89 +ibgda +nccl +ucx +mpi' + + openmpi: + require: + - spec: '+cuda cuda_arch=89 target=zen4' + + petsc: + require: + - spec: '+cuda cuda_arch=89' + when: '%gcc' + + py-tensorflow: + require: + - spec: '+cuda cuda_arch=89 +nccl' + + py-torch: + require: + - spec: '+cuda cuda_arch=89 +nccl +cudnn' + + quantum-espresso: + require: + - spec: '~cuda' + + suite-sparse: + require: + - spec: '+cuda' + + ucx: + require: + - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=89 ~rocm target=zen4' + + # --------------------------------------------------------------------------- + # Externals + # --------------------------------------------------------------------------- + pmix: + buildable: false + externals: + - spec: 'pmix@5.0.2' + prefix: /usr + + rdma-core: + buildable: false + externals: + - spec: 'rdma-core@51.1' + prefix: /usr + + slurm: + buildable: false + externals: + - spec: 'slurm@24-05-0-2' + prefix: /usr + + xpmem: + buildable: false + externals: + - spec: 'xpmem@2.7.3' + prefix: /usr diff --git a/stacks/pinot-noir/packages_stack.yaml b/stacks/pinot-noir/packages_stack.yaml index d12a3aa85defa2721cc41fe5bb6811632456b7d7..65bebbff72b75fc05be003c6c16492fbf27e07a0 100644 --- a/stacks/pinot-noir/packages_stack.yaml +++ b/stacks/pinot-noir/packages_stack.yaml @@ -6,9 +6,6 @@ packages: jpeg: [libjpeg] zlib-api: [zlib-ng] require: - - spec: "^libiconv" - when: "^iconv" - - spec: '+ipo' when: '%gcc' @@ -133,9 +130,6 @@ packages: require: - spec: '+rdmacm +rc +dc +ud +cma +verbs' - zlib-ng: - require: ['build_system=autotools'] - # ---------------------------------------------------------------------------- # System dependencies # ---------------------------------------------------------------------------- diff --git a/stacks/pinot-noir/spack.yaml b/stacks/pinot-noir/spack.yaml index d80e2cc8ab5f37da522e571dfb1818155c18598c..4450501f931976ff7626285308a5700d39fdbc5a 100644 --- a/stacks/pinot-noir/spack.yaml +++ b/stacks/pinot-noir/spack.yaml @@ -6,6 +6,7 @@ spack: - packages_env.yaml - modules_stack.yaml + - modules_env.yaml - definitions_env.yaml