diff --git a/ci/install_compilers.sh b/ci/install_compilers.sh
index 3c5547b74d6b9829b5754e8be818dab448d0410d..fbfeef455ccdf0d086f757426481a20f2c670f72 100755
--- a/ci/install_compilers.sh
+++ b/ci/install_compilers.sh
@@ -73,13 +73,6 @@ for c in "core_compilers" "compilers"; do
fi
done
-export target=$(jq -Mrc '
-.stack.targets
- | to_entries
- | .[]
- | select(.key == env.environment)
- | .value' ${STACK_CONFIG})
-
compilers_specs=$(jq -Mr '
.stack
| .system_arch as $arch
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index cbaa200f92444dbf7bad98514f53f67a2f72636f..5881e70b45f524f72e69573af0a072f5da35665d 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -26,30 +26,48 @@ then
spack/bin/spack env create ${environment}
fi
-for file in mirrors packages concretizer config spack definitions modules
-do
- if [ -e "${STACK_CONFIG_PATH}/${file}.yaml" ]; then
- echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
- cp "${STACK_CONFIG_PATH}/${file}.yaml" \
- ${SPACK_SYSTEM_CONFIG_PATH}
- cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
- fi
-
- if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
- echo "Copying ${STACK_CONFIG_PATH}/${file}_stack.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
- cp "${STACK_CONFIG_PATH}/${file}_stack.yaml" \
- ${SPACK_SYSTEM_CONFIG_PATH}
- fi
- if [ -e "${STACK_CONFIG_PATH}/${file}_${environment}.yaml" ]; then
- echo "Copying ${STACK_CONFIG_PATH}/${file}_${environment}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}/${file}_env.yaml"
- cp "${STACK_CONFIG_PATH}/${file}_${environment}.yaml" \
- ${SPACK_SYSTEM_CONFIG_PATH}/${file}_env.yaml
- fi
-done
+echo "Copying cp ${STACK_CONFIG_PATH}/spack.yaml to ${SPACK_SYSTEM_CONFIG_PATH}/"
+cp ${STACK_CONFIG_PATH}/spack.yaml ${SPACK_SYSTEM_CONFIG_PATH}
cp -r "${STACK_CONFIG_PATH}/templates" ${SPACK_SYSTEM_CONFIG_PATH}
+declare -A file_mapping
+file_mapping["_common"]=""
+file_mapping["_stack"]="_stack"
+file_mapping["_acc_${acc_type}"]="_accel"
+file_mapping["_${environment}"]="_env"
+
+declare -A placeholders
+placeholders["{target}"]="${target}"
+placeholders["{gcc_version}"]=$(jq -Mrc '.stack.compilers.gcc.version' ${STACK_CONFIG})
+if [ "${accelerator}" != "none" ]
+then
+ placeholders["{${acc_type}_arch}"]="${acc_arch}"
+fi
+
+echo blip
+
+for file in mirrors packages concretizer config definitions modules
+do
+ for suffix in ${!file_mapping[@]}
+ do
+ src=${STACK_CONFIG_PATH}/${file}${suffix}.yaml
+ dst=${SPACK_SYSTEM_CONFIG_PATH}/${file}"${file_mapping[$suffix]}".yaml
+ if [ -e "${STACK_CONFIG_PATH}/${file}${suffix}.yaml" ]
+ then
+ echo "Copying $src to $dst"
+ cp "$src" "$dst"
+
+ for ph in ${!placeholders[@]}
+ do
+ rep=${placeholders[$ph]}
+ sed -i -e "s/$ph/$rep/g" "$dst"
+ done
+ fi
+ done
+done
+
cat <<EOF > ${SPACK_SYSTEM_CONFIG_PATH}/config_stack.yaml
config:
@@ -78,7 +96,6 @@ mirrors:
${mirrors}
EOF
-cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
echo "Setting up packages"
@@ -101,18 +118,6 @@ export system_arch=$(jq -Mrc '
.stack.system_arch
' ${STACK_CONFIG})
-# cat <<EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-# all:
-# prefer:
-# EOF
-
-# jq -Mrc '
-# .stack.system_packages
-# | map(" - spec: \"^\(.)%\(env.system_compiler)\"\n when: \"%nvhpc ^\(.)\"")
-# | .[]
-# ' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-
jq -Mrc '
.stack.system_packages
| map(" \(.):\n require:\n - \"%\(env.system_compiler)\"\n - \"\(env.system_arch)\"")
@@ -126,8 +131,6 @@ then
rm packages.yaml.old
fi
-cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
-
echo "Setting up extra repos"
EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
@@ -155,7 +158,14 @@ do
fi
done
-cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
+for file in mirrors packages concretizer config modules repos
+do
+ if [ -e "${SPACK_SYSTEM_CONFIG_PATH}/${file}.yaml" ]
+ then
+ cp "${SPACK_SYSTEM_CONFIG_PATH}/${file}.yaml" spack/etc/spack
+ fi
+done
+
echo "Setting up buildcache"
diff --git a/ci/stack_env.sh b/ci/stack_env.sh
index aac06691ab1680381527903fc8e26303eea64f5b..a08ab7cd8ecc983a50c86d7427ca0bb9da3ee4a2 100644
--- a/ci/stack_env.sh
+++ b/ci/stack_env.sh
@@ -14,6 +14,18 @@ export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/spack/var/spack/environments/$
#export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)
export environment_type="local_cluster"
+export SPACK_VERSION=$(jq -r .spack.version ${STACK_CONFIG})
+
+env_json=$(jq '.stack.environments | to_entries | .[] | select(.key == env.environment) | .value' ${STACK_CONFIG})
+
+export target=$(echo $env_json | jq -Mrc '.target')
+export accelerator=$(echo $env_json | jq -Mrc '.accelerator // "none"')
+
+if [ "$accelerator" != "none" ]
+then
+ export acc_type=$(echo $env_json | jq -Mrc '.accelerator.type')
+ export acc_arch=$(echo $env_json | jq -Mrc '.accelerator.arch')
+fi
echo "STACK_CONFIG_PATH: ${STACK_CONFIG_PATH}"
echo "STACK_CONFIG: ${STACK_CONFIG}"
diff --git a/stacks/pinot-noir-gcc/concretizer.yaml b/stacks/pinot-noir-gcc/concretizer_common.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/concretizer.yaml
rename to stacks/pinot-noir-gcc/concretizer_common.yaml
diff --git a/stacks/pinot-noir-gcc/config.yaml b/stacks/pinot-noir-gcc/config_common.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/config.yaml
rename to stacks/pinot-noir-gcc/config_common.yaml
diff --git a/stacks/pinot-noir-gcc/definitions_cuda.yaml b/stacks/pinot-noir-gcc/definitions_acc_cuda.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/definitions_cuda.yaml
rename to stacks/pinot-noir-gcc/definitions_acc_cuda.yaml
diff --git a/stacks/pinot-noir-gcc/definitions_empty.yaml b/stacks/pinot-noir-gcc/definitions_acc_none.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/definitions_empty.yaml
rename to stacks/pinot-noir-gcc/definitions_acc_none.yaml
diff --git a/stacks/pinot-noir-gcc/definitions_izar.yaml b/stacks/pinot-noir-gcc/definitions_izar.yaml
deleted file mode 120000
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_izar.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma-h100.yaml b/stacks/pinot-noir-gcc/definitions_kuma-h100.yaml
deleted file mode 120000
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_kuma-h100.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma-l40s.yaml b/stacks/pinot-noir-gcc/definitions_kuma-l40s.yaml
deleted file mode 120000
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_kuma-l40s.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/mirrors.yaml b/stacks/pinot-noir-gcc/mirrors.yaml
deleted file mode 100644
index 2c338aa265ca4ab7735374e44dd491d1f2c81115..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/mirrors.yaml
+++ /dev/null
@@ -1 +0,0 @@
-mirrors: {}
diff --git a/stacks/pinot-noir-gcc/modules_empty.yaml b/stacks/pinot-noir-gcc/modules_acc_none.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/modules_empty.yaml
rename to stacks/pinot-noir-gcc/modules_acc_none.yaml
diff --git a/stacks/pinot-noir-gcc/modules.yaml b/stacks/pinot-noir-gcc/modules_common.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/modules.yaml
rename to stacks/pinot-noir-gcc/modules_common.yaml
diff --git a/stacks/pinot-noir-gcc/packages.yaml b/stacks/pinot-noir-gcc/packages_common.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/packages.yaml
rename to stacks/pinot-noir-gcc/packages_common.yaml
diff --git a/stacks/pinot-noir/concretizer.yaml b/stacks/pinot-noir/concretizer_common.yaml
similarity index 100%
rename from stacks/pinot-noir/concretizer.yaml
rename to stacks/pinot-noir/concretizer_common.yaml
diff --git a/stacks/pinot-noir/config.json b/stacks/pinot-noir/config.json
index 5a5c053d79d46ea02bdb93f6bd438c920457f927..174e76651437e2fb2a016aa9243c3e0237c4fe5a 100644
--- a/stacks/pinot-noir/config.json
+++ b/stacks/pinot-noir/config.json
@@ -33,12 +33,35 @@
"version": "11.2.1"
}
},
- "targets": {
- "helvetios": "skylake_avx512",
- "izar": "cascadelake",
- "jed": "icelake",
- "kuma-l40s": "zen4",
- "kuma-h100": "zen4"
+ "environments": {
+ "helvetios": {
+ "target": "skylake_avx512",
+ "accelerator": "none"
+ },
+ "izar": {
+ "target": "cascadelake",
+ "accelerator": {
+ "type": "cuda",
+ "arch": "75"
+ }
+ },
+ "jed": {
+ "target": "icelake"
+ },
+ "kuma-l40s": {
+ "target": "zen4",
+ "accelerator": {
+ "type": "cuda",
+ "arch": "89"
+ }
+ },
+ "kuma-h100": {
+ "target": "zen4",
+ "accelerator": {
+ "type": "cuda",
+ "arch": "90"
+ }
+ }
},
"compilers": {
"oneapi": {
diff --git a/stacks/pinot-noir/config.yaml b/stacks/pinot-noir/config_common.yaml
similarity index 100%
rename from stacks/pinot-noir/config.yaml
rename to stacks/pinot-noir/config_common.yaml
diff --git a/stacks/pinot-noir/definitions_cuda.yaml b/stacks/pinot-noir/definitions_acc_cuda.yaml
similarity index 100%
rename from stacks/pinot-noir/definitions_cuda.yaml
rename to stacks/pinot-noir/definitions_acc_cuda.yaml
diff --git a/stacks/pinot-noir-gcc/definitions_helvetios.yaml b/stacks/pinot-noir/definitions_acc_none.yaml
similarity index 100%
rename from stacks/pinot-noir-gcc/definitions_helvetios.yaml
rename to stacks/pinot-noir/definitions_acc_none.yaml
diff --git a/stacks/pinot-noir/definitions_izar.yaml b/stacks/pinot-noir/definitions_izar.yaml
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..3552a78f67656b1b6647cced51cbfb18b5882f10 120000
--- a/stacks/pinot-noir/definitions_izar.yaml
+++ b/stacks/pinot-noir/definitions_izar.yaml
@@ -1 +1 @@
-definitions_cuda.yaml
\ No newline at end of file
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_jed.yaml b/stacks/pinot-noir/definitions_jed.yaml
deleted file mode 100644
index 0f156fa830b9567b3e7839218a0a88c16076c6ca..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir/definitions_jed.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir/definitions_jed.yaml b/stacks/pinot-noir/definitions_jed.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..3552a78f67656b1b6647cced51cbfb18b5882f10
--- /dev/null
+++ b/stacks/pinot-noir/definitions_jed.yaml
@@ -0,0 +1 @@
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_kuma-h100.yaml b/stacks/pinot-noir/definitions_kuma-h100.yaml
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..3552a78f67656b1b6647cced51cbfb18b5882f10 120000
--- a/stacks/pinot-noir/definitions_kuma-h100.yaml
+++ b/stacks/pinot-noir/definitions_kuma-h100.yaml
@@ -1 +1 @@
-definitions_cuda.yaml
\ No newline at end of file
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/definitions_kuma-l40s.yaml b/stacks/pinot-noir/definitions_kuma-l40s.yaml
index 2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f..3552a78f67656b1b6647cced51cbfb18b5882f10 120000
--- a/stacks/pinot-noir/definitions_kuma-l40s.yaml
+++ b/stacks/pinot-noir/definitions_kuma-l40s.yaml
@@ -1 +1 @@
-definitions_cuda.yaml
\ No newline at end of file
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/mirrors.yaml b/stacks/pinot-noir/mirrors.yaml
deleted file mode 100644
index 2c338aa265ca4ab7735374e44dd491d1f2c81115..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir/mirrors.yaml
+++ /dev/null
@@ -1 +0,0 @@
-mirrors: {}
diff --git a/stacks/pinot-noir/modules_acc_cuda.yaml b/stacks/pinot-noir/modules_acc_cuda.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6baee16b9a1286c8af13e7e5ba223b08c96f2d4f
--- /dev/null
+++ b/stacks/pinot-noir/modules_acc_cuda.yaml
@@ -0,0 +1,21 @@
+modules:
+ default:
+ lmod:
+ nvhpc+mpi@24.3:
+ environment:
+ set:
+ HPCX_DIR: "{prefix}/Linux_x86_64/24.3/comm_libs/12.3/hpcx/hpcx-2.17.1"
+ OPAL_PREFIX: "${HPCX_DIR}/ompi"
+ prepend_path:
+ LD_LIBRARY_PATH: "${HPCX_DIR}/ompi/lib"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/ucx/lib"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/ucx/lib/ucx"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/ucc/lib"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/ucc/lib/ucc"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/hcoll/lib"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/sharp/lib"
+ LD_LIBRARY_PATH: "${HPCX_DIR}/nccl_rdma_sharp_plugin/lib"
+ PATH: "${HPCX_DIR}/ucx/bin"
+ PATH: "${HPCX_DIR}/ucc/bin"
+ PATH: "${HPCX_DIR}/hcoll/bin"
+ PATH: "${HPCX_DIR}/sharp/bin"
diff --git a/stacks/pinot-noir/modules_acc_none.yaml b/stacks/pinot-noir/modules_acc_none.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4
--- /dev/null
+++ b/stacks/pinot-noir/modules_acc_none.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir/modules.yaml b/stacks/pinot-noir/modules_common.yaml
similarity index 92%
rename from stacks/pinot-noir/modules.yaml
rename to stacks/pinot-noir/modules_common.yaml
index 38fcc67d3399d00e3662efcae31e385da1fb39ad..08558927791b43b6741f8126d6d529439f2e056c 100644
--- a/stacks/pinot-noir/modules.yaml
+++ b/stacks/pinot-noir/modules_common.yaml
@@ -3,10 +3,13 @@ modules:
enable: [lmod]
lmod:
hash_length: 0
+
core_compilers:
- gcc@11.2.1
+
hierarchy:
- mpi
+
all:
environment:
set:
@@ -40,53 +43,63 @@ modules:
F77: "{prefix}/bin/gfortran"
FC: "{prefix}/bin/gfortran"
F90: "{prefix}/bin/gfortran"
- cuda@12.4.0:
- environment:
- append_path:
- JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/cuda"
+
+# cuda:
+# environment:
+# append_path:
+# JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/cuda"
openmpi:
environment:
set:
SLURM_MPI_TYPE: pmix
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
- append_path:
- JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
+# append_path:
+# JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
+
openblas threads=pthreads:
environment:
set:
OPENBLAS_NUM_THREADS: '1'
+
hdf5~mpi+cxx:
environment:
set:
HDF5_CC: h5cc
HDF5_CXX: h5c++
HDF5_FC: h5fc
+
hdf5+mpi~cxx:
environment:
set:
HDF5_CC: h5pcc
HDF5_FC: h5pfc
+
intel-oneapi-mpi:
environment:
set:
I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so
I_MPI_EXTRA_FILESYSTEM: '0'
SLURM_MPI_TYPE: pmi2
+
metis:
suffixes:
~real64: sp
+
molpro:
template: modules/group_restricted.lua
+
plumed:
environment:
set:
PLUMED_KERNEL: "{prefix}/lib/libplumedKernel.so"
unset:
- PLUMED_ROOT
+
quantum-espresso:
suffixes:
hdf5=parallel: hdf5
+
scons:
suffixes:
'^python@:2.99': py2
@@ -94,16 +107,21 @@ modules:
abaqus:
template: modules/group_restricted.lua
+
adf:
template: modules/group_restricted.lua
+
ams:
template: modules/group_restricted.lua
+
comsol:
environment:
prepend_path:
MATLABPATH: "{prefix}/mli"
+
crystal17:
template: modules/group_restricted.lua
+
fdtd:
environment:
prepend_path:
@@ -111,12 +129,15 @@ modules:
LD_LIBRARY_PATH: "{prefix}/mpich2/nemesis/lib"
set:
SLURM_MPI_TYPE: pmi2
+
gaussian:
template: modules/group_restricted.lua
+
spark:
environment:
prepend_path:
PATH: "/ssoft/spack/scripts/all/spark"
+
ucx:
environment:
unset:
diff --git a/stacks/pinot-noir/modules_kuma-h100.yaml b/stacks/pinot-noir/modules_kuma-h100.yaml
index 4599ea4a749e4ffd3b6fcdb239b836e8fb83a7c7..8ddd771fff8e49d8a00ac2ba17096cdd8ff145ed 100644
--- a/stacks/pinot-noir/modules_kuma-h100.yaml
+++ b/stacks/pinot-noir/modules_kuma-h100.yaml
@@ -12,3 +12,8 @@ modules:
set:
NCCL_IB_HCA: 'mlx5_2:1,mlx5_3:1'
NCCL_SOCKET_IFNAME: 'bond0'
+ nvshmem:
+ environment:
+ set:
+ NVSHMEM_HCA_LIST: 'mlx5_2:1,mlx5_3:1'
+ NVSHMEM_BOOTSTRAP: MPI
diff --git a/stacks/pinot-noir/modules_kuma-l40s.yaml b/stacks/pinot-noir/modules_kuma-l40s.yaml
index 1f147c1ca09b032013ea477d57be60daa8e98703..10a02d6a9cb56d2f1bcb9e66730d60c124561bda 100644
--- a/stacks/pinot-noir/modules_kuma-l40s.yaml
+++ b/stacks/pinot-noir/modules_kuma-l40s.yaml
@@ -12,3 +12,8 @@ modules:
set:
NCCL_IB_HCA: 'mlx5_0:1,mlx5_1:1'
NCCL_SOCKET_IFNAME: 'bond0'
+ nvshmem:
+ environment:
+ set:
+ NVSHMEM_HCA_LIST: 'mlx5_0:1,mlx5_1:1'
+ NVSHMEM_BOOTSTRAP: MPI
diff --git a/stacks/pinot-noir/packages_acc_cuda.yaml b/stacks/pinot-noir/packages_acc_cuda.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d531baa920b6e310d21015c906910690ed1de541
--- /dev/null
+++ b/stacks/pinot-noir/packages_acc_cuda.yaml
@@ -0,0 +1,86 @@
+packages:
+ all:
+ require:
+ - "~rocm"
+
+ - spec: cuda_arch={cuda_arch}
+ when: '+cuda'
+
+ - spec: '+cuda'
+ when: '%gcc'
+
+ # ---------------------------------------------------------------------------
+ cuda:
+ prefer:
+ - spec: "^gcc-runtime@{gcc_version}"
+ when: "%gcc"
+ - spec: "%gcc@{gcc_version} target={target}"
+ when: "%gcc"
+ require:
+ - '@12.4.1'
+
+ hypre:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch} +unified-memory'
+ when: '%gcc'
+
+ kokkos:
+ require:
+ - spec: '+cuda +cuda_lambda cuda_arch={cuda_arch} +wrapper ~openmptarget'
+ when: '%gcc'
+
+ kokkos-kernels:
+ require: ['+cuda', '+cublas', '+cusolver', '+cusparse', 'cuda_arch={cuda_arch}']
+
+ libxc:
+ require: ['+cuda', 'cuda_arch={cuda_arch}']
+
+ nvshmem:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch} +ibgda +nccl +ucx +mpi'
+
+ openmpi:
+ require: ['+cuda', 'cuda_arch={cuda_arch}', 'fabrics=hcoll']
+
+ papi:
+ require: ['+nvml', '+infiniband', '+cuda']
+
+ paraview:
+ require: ['~cuda']
+
+ petsc:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch}'
+ when: '%gcc'
+
+ py-tensorflow:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch} +nccl'
+
+ py-torch:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch} +nccl +cudnn'
+
+ python:
+ require:
+ - spec: 'target={target}'
+ when: '%gcc@{gcc_version}'
+
+ quantum-espresso:
+ require:
+ - spec: '+nvtx'
+
+ suite-sparse:
+ require:
+ - spec: '+cuda'
+ when: '@7.3.1'
+ - spec: '~cuda'
+ when: '@7.2.1'
+
+ ucc:
+ require:
+ - spec: '+cuda cuda_arch={cuda_arch} +nccl'
+
+ ucx:
+ require:
+ - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch={cuda_arch} ~rocm'
diff --git a/stacks/pinot-noir/packages_acc_none.yaml b/stacks/pinot-noir/packages_acc_none.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0aa29d8aa89378efccc88721d31443a6c52c5595
--- /dev/null
+++ b/stacks/pinot-noir/packages_acc_none.yaml
@@ -0,0 +1,3 @@
+packages:
+ all:
+ require: ['~cuda', '~rocm']
diff --git a/stacks/pinot-noir/packages.yaml b/stacks/pinot-noir/packages_common.yaml
similarity index 100%
rename from stacks/pinot-noir/packages.yaml
rename to stacks/pinot-noir/packages_common.yaml
diff --git a/stacks/pinot-noir/packages_helvetios.yaml b/stacks/pinot-noir/packages_helvetios.yaml
deleted file mode 100644
index fe0d4ad01829f0895adce34a9071fe0eb4218e18..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir/packages_helvetios.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-packages:
- all:
- target: ["skylake_avx512"]
- require: ['~cuda', '~rocm']
-
- # py-torch:
- # require:
- # - spec: '~cuda ~cudnn ~nccl ~rocm'
diff --git a/stacks/pinot-noir/packages_izar.yaml b/stacks/pinot-noir/packages_izar.yaml
deleted file mode 100644
index 9ce65d95e200925fe7b5bd50d0f76c18b2f33097..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir/packages_izar.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-packages:
- all:
- target: ["cannonlake"]
- require:
- - spec: ^openmpi +cuda
- when: '%gcc ^mpi'
-
- - spec: cuda_arch=70
- when: '+cuda'
-
- - '+cuda'
diff --git a/stacks/pinot-noir/packages_jed.yaml b/stacks/pinot-noir/packages_jed.yaml
deleted file mode 100644
index 822d1e2f6b56828d7269ba821ecb27be99c9d6d2..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir/packages_jed.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-packages:
- all:
- target: ["icelake"]
- require:
- - "~rocm"
-
- - spec: cuda_arch=90 target=icelake
- when: '+cuda'
- # - spec: '~cuda'
- # when: '%oneapi'
-
- - spec: '+cuda'
- when: '%gcc'
-
- # ---------------------------------------------------------------------------
- hypre:
- require:
- - spec: '+cuda cuda_arch=90 +unified-memory'
- when: '%gcc'
-
- kokkos:
- require:
- - spec: '+cuda cuda_arch=90 +cuda_uvm +wrapper ~openmptarget'
- when: '%gcc'
-
- kokkos-kernels:
- require:
- - spec: '+cuda cuda_arch=90'
- when: '%gcc'
-
- openmpi:
- require:
- - spec: '+cuda cuda_arch=90 target=icelake'
-
- petsc:
- require:
- - spec: '+cuda cuda_arch=90'
- when: '%gcc'
-
- py-tensorflow:
- require:
- - spec: '+cuda cuda_arch=90 +nccl'
-
- py-torch:
- require:
- - spec: '+cuda cuda_arch=90 +nccl +cudnn'
-
- quantum-espresso:
- require:
- - spec: '~cuda'
-
- suite-sparse:
- require:
- - spec: '+cuda'
-
- ucx:
- require:
- - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=90 target=icelake ~rocm'
-
- # ---------------------------------------------------------------------------
- # Externals
- # ---------------------------------------------------------------------------
- pmix:
- buildable: false
- externals:
- - spec: 'pmix@5.0.2'
- prefix: /usr
-
- rdma-core:
- buildable: false
- externals:
- - spec: 'rdma-core@47.1'
- prefix: /usr
-
- slurm:
- buildable: false
- externals:
- - spec: 'slurm@24-05-0-2'
- prefix: /usr
-
- xpmem:
- buildable: false
- externals:
- - spec: 'xpmem@2.7.3'
- prefix: /usr
diff --git a/stacks/pinot-noir/packages_kuma-h100.yaml b/stacks/pinot-noir/packages_kuma-h100.yaml
index 516d8174918e38643f5548077b13c68b79daf375..fb5764966000dd33ff469f833f1851d7fb1daeb0 100644
--- a/stacks/pinot-noir/packages_kuma-h100.yaml
+++ b/stacks/pinot-noir/packages_kuma-h100.yaml
@@ -1,94 +1,16 @@
packages:
- all:
- target: ["zen4"]
- prefer: ['target=zen4']
- require:
- - "~rocm"
-
- - spec: cuda_arch=90
- when: '+cuda'
-
- - spec: '+cuda'
- when: '%gcc'
-
- # ---------------------------------------------------------------------------
cp2k:
require: ['+cuda', 'cuda_arch=90']
- cuda:
- prefer:
- - spec: "^gcc-runtime@13.2.0"
- when: "%gcc"
- - spec: "%gcc@13.2.0 target=zen4"
- when: "%gcc"
- require:
- - '@12.4.1'
-
- hypre:
- require:
- - spec: '+cuda cuda_arch=90 +unified-memory'
- when: '%gcc'
-
- kokkos:
- require:
- - spec: '+cuda +cuda_lambda cuda_arch=90 +wrapper ~openmptarget'
- when: '%gcc'
-
- kokkos-kernels:
- require: ['+cuda', '+cublas', '+cusolver', '+cusparse', 'cuda_arch=90']
-
- libxc:
- require: ['+cuda', 'cuda_arch=90']
-
- nvshmem:
- require:
- - spec: '+cuda cuda_arch=90 +ibgda +nccl +ucx +mpi'
-
- openmpi:
- require: ['+cuda', 'cuda_arch=90']
-
- papi:
- require: ['+nvml', '+infiniband', '+cuda']
-
- paraview:
- require: ['~cuda']
-
- petsc:
- require:
- - spec: '+cuda cuda_arch=90'
- when: '%gcc'
-
- py-tensorflow:
- require:
- - spec: '+cuda cuda_arch=90 +nccl'
-
- py-torch:
- require:
- - spec: '+cuda cuda_arch=90 +nccl +cudnn'
-
- python:
- require:
- - spec: 'target=zen4'
- when: '%gcc@13.2.0'
-
- quantum-espresso:
- require:
- - spec: '+nvtx'
-
- suite-sparse:
- require:
- - spec: '+cuda'
- when: '@7.3.1'
- - spec: '~cuda'
- when: '@7.2.1'
-
- ucx:
- require:
- - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=90 ~rocm'
-
# ---------------------------------------------------------------------------
# Externals
# ---------------------------------------------------------------------------
+ hcoll:
+ buildable: false
+ externals:
+ - spec: 'hcoll@4.8.3227'
+ prefix: /opt/mellanox/hcoll
+
pmix:
buildable: false
externals:
diff --git a/stacks/pinot-noir/packages_kuma-l40s.yaml b/stacks/pinot-noir/packages_kuma-l40s.yaml
index 323b8b2a12d01ea19517423ef5ffa32dbc9bba9d..bfde5dc545e618026028c5d89dba4fb2d7397f01 100644
--- a/stacks/pinot-noir/packages_kuma-l40s.yaml
+++ b/stacks/pinot-noir/packages_kuma-l40s.yaml
@@ -1,85 +1,8 @@
packages:
- all:
- target: ["zen4"]
- prefer: ["target=zen4"]
- require:
- - "~rocm"
-
- - spec: cuda_arch=89
- when: '+cuda'
-
- - spec: '+cuda'
- when: '%gcc'
-
# ---------------------------------------------------------------------------
- cuda:
- prefer:
- - spec: "^gcc-runtime@13.2.0"
- when: "%gcc"
- - spec: "%gcc@13.2.0 target=zen4"
- when: "%gcc"
- require:
- - '@12.4.1'
-
- hypre:
- require:
- - spec: '+cuda cuda_arch=89 +unified-memory'
- when: '%gcc'
-
- kokkos:
- require:
- - spec: '+cuda +cuda_lambda cuda_arch=89 +wrapper ~openmptarget'
- when: '%gcc'
-
- kokkos-kernels:
- require: ['+cuda', '+cublas', '+cusolver', '+cusparse', 'cuda_arch=89']
-
- nvshmem:
- require:
- - spec: '+cuda cuda_arch=89 +ibgda +nccl +ucx +mpi'
-
- openmpi:
- require: ['+cuda', 'cuda_arch=89']
-
- papi:
- require: ['+nvml', '+infiniband', '+cuda']
-
- paraview:
+ cp2k:
require: ['~cuda']
- petsc:
- require:
- - spec: '+cuda cuda_arch=89'
- when: '%gcc'
-
- py-tensorflow:
- require:
- - spec: '+cuda cuda_arch=89 +nccl'
-
- py-torch:
- require:
- - spec: '+cuda cuda_arch=89 +nccl +cudnn'
-
- python:
- require:
- - spec: 'target=zen4'
- when: '%gcc@13.2.0'
-
- quantum-espresso:
- require:
- - spec: '+nvtx'
-
- suite-sparse:
- require:
- - spec: '+cuda'
- when: '@7.3.1'
- - spec: '~cuda'
- when: '@7.2.1'
-
- ucx:
- require:
- - spec: '+xpmem +cma +rdmacm +rc +ud +dc +verbs +ib_hw_tm +gdrcopy +cuda cuda_arch=89 ~rocm'
-
# ---------------------------------------------------------------------------
# Externals
# ---------------------------------------------------------------------------
diff --git a/stacks/pinot-noir/packages_stack.yaml b/stacks/pinot-noir/packages_stack.yaml
index 5ca72c697f9376ee1abada95ec2c555fd679e331..a5b0f165221bba47bfcbed2fd098effd0ce0d985 100644
--- a/stacks/pinot-noir/packages_stack.yaml
+++ b/stacks/pinot-noir/packages_stack.yaml
@@ -1,8 +1,12 @@
packages:
all:
+ target: ["{target}"]
+ prefer: ['target={target}']
+
require:
- spec: '+ipo'
when: '%gcc'
+
- spec: '^netlib-scalapack'
when: '%gcc ^scalapack'
@@ -191,7 +195,7 @@ packages:
openmpi:
require: ['@5.0.3',
- 'fabrics=ofi,ucx,verbs',
+ 'fabrics=ofi,ucc,ucx,verbs',
'schedulers=slurm',
'+romio', 'romio-filesystem=gpfs',
'+internal-pmix', '~rsh',
@@ -290,8 +294,7 @@ packages:
require: ['+xft', '+xss']
ucx:
- require:
- - spec: '+rdmacm +rc +dc +ud +cma +verbs'
+ require: ['@1.17.0', '+rdmacm', '+rc', '+dc', '+ud', '+cma', '+verbs']
vtk:
require:
@@ -339,7 +342,6 @@ packages:
- spec: 'slurm@23-11-7-1'
prefix: /usr
-
# ----------------------------------------------------------------------------
# Externals
# ----------------------------------------------------------------------------
diff --git a/stacks/pinot-noir/spack.yaml b/stacks/pinot-noir/spack.yaml
index cef8b490bd560a5c07f1be5060cd7299d6cae071..bb0a9a7515a2d3058fc1285e3df0f6d8147bd5e6 100644
--- a/stacks/pinot-noir/spack.yaml
+++ b/stacks/pinot-noir/spack.yaml
@@ -3,11 +3,13 @@ spack:
- compiler_definitions.yaml
- packages_stack.yaml
+ - packages_accel.yaml
- packages_env.yaml
- modules_stack.yaml
- modules_env.yaml
+ - definitions_accel.yaml
- definitions_env.yaml
- config_stack.yaml