diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d97b0df0c0dc9a74fc6295aa270607a0070b6203..aa71e1603c9511fe04e3ffca505ea92cfecea1e9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -90,33 +90,17 @@ spack:setup:
- ci/setup_spack.sh
needs:
- job: spack:checkout
- after_script:
- - tar czf artifacts_${stack}_${environment}.tgz $stack/environment
- artifacts:
- paths:
- - artifacts_${stack}_${environment}.tgz
spack:install_compilers:
stage: compilers
extends:
- .parallel_job
- .spack_cache
- before_script:
- - if [ -e artifacts_${stack}_${environment}.tgz ];
- then
- tar xzf artifacts_${stack}_${environment}.tgz;
- fi
script:
- ci/install_compilers.sh
- source ci/stack_env.sh
- ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame compilers
- after_script:
- - tar czf
- artifacts_${stack}_${environment}.tgz
- $stack/spack/var/spack/environments
artifacts:
- paths:
- - artifacts_${stack}_${environment}.tgz
reports:
junit: spack-install-*.xml
needs:
@@ -128,36 +112,23 @@ spack:concretize:
extends:
- .parallel_job
- .spack_cache
- before_script:
- - if [ -e artifacts_${stack}_${environment}.tgz ];
- then
- tar xzf artifacts_${stack}_${environment}.tgz;
- fi
script:
- source ci/stack_env.sh
- - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame
- - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force
- after_script:
- - tar czf
- artifacts_${stack}_${environment}.tgz
- $stack/spack/var/spack/environments
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame | tee config.log
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force | tee concretize.log
artifacts:
paths:
- - artifacts_${stack}_${environment}.tgz
+ - config.log
+ - concretize.log
needs:
- job: spack:install_compilers
- timeout: 5h
+ timeout: 24h
spack:install:
stage: install
extends:
- .parallel_job
- .spack_cache
- before_script:
- - if [ -e artifacts_${stack}_${environment}.tgz ];
- then
- tar xzf artifacts_${stack}_${environment}.tgz;
- fi
script:
- source ci/stack_env.sh
- echo "{}" > spack-install-${environment}.xml
@@ -174,18 +145,12 @@ spack:install:
--update-index
--key EDC904DCE3D2E84E
/buildcache
- after_script:
- - tar czf
- artifacts_${stack}_${environment}.tgz
- $stack/spack/var/spack/environments
needs:
- job: spack:concretize
artifacts:
- paths:
- - artifacts_${stack}_${environment}.tgz
reports:
junit: spack-install-*.xml
- timeout: 10h
+ timeout: 72h
spack:mksquashfs:
stage: deploy
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index 81af918eca15d17723124e81824a21c8125de471..f86ed7e0d538403fc45113f413f68a8a0d095db2 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -26,12 +26,13 @@ then
spack/bin/spack env create ${environment}
fi
-for file in mirrors packages concretizer config spack definitions
+for file in mirrors packages concretizer config spack definitions modules
do
if [ -e "${STACK_CONFIG_PATH}/${file}.yaml" ]; then
echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
cp "${STACK_CONFIG_PATH}/${file}.yaml" \
${SPACK_SYSTEM_CONFIG_PATH}
+ cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
fi
if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
@@ -77,6 +78,28 @@ mirrors:
${mirrors}
EOF
+cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
+
+echo "Setting up packages"
+
+echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+export system_compiler=$(jq -Mrc '
+.stack
+ | .system_arch as $arch
+ | .system_compiler
+ | to_entries
+ | .[].value
+ | "\(.compiler)@\(.version) \($arch)"
+' ${STACK_CONFIG})
+
+jq -Mrc '
+.stack.system_packages
+ | map(" \(.):\n require:\n - spec: \"%\(env.system_compiler)\"")
+ | .[]
+' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
+
echo "Setting up extra repos"
EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
@@ -104,6 +127,8 @@ do
fi
done
+cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
+
echo "Setting up buildcache"
spack/bin/spack gpg trust \
@@ -127,19 +152,3 @@ spack/bin/spack buildcache keys \
--trust
#spack/bin/spack buildcache update-index /buildcache
-
-echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-export system_compiler=$(jq -Mrc '
-.stack
- | .system_arch as $arch
- | .system_compiler
- | to_entries
- | .[].value
- | "\(.compiler)@\(.version) \($arch)"
-' ${STACK_CONFIG})
-
-jq -Mrc '
-.stack.system_packages
- | map(" \(.):\n require:\n - spec: \"%\(env.system_compiler)\"")
- | .[]
-' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
diff --git a/stacks/pinot-noir-gcc/config.yaml b/stacks/pinot-noir-gcc/config.yaml
index ca9196448b45d5047a79a6e262ddea51f6d45214..b0e3ad831aaf6ec6c2954e120263b3da6459499a 100644
--- a/stacks/pinot-noir-gcc/config.yaml
+++ b/stacks/pinot-noir-gcc/config.yaml
@@ -3,7 +3,7 @@ config:
install_missing_compilers: false
install_tree:
- padded_length: 227
+ padded_length: 200
build_stage:
- $tempdir/$user/spack-stage
diff --git a/stacks/pinot-noir-gcc/definitions_cuda.yaml b/stacks/pinot-noir-gcc/definitions_cuda.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0f156fa830b9567b3e7839218a0a88c16076c6ca
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_cuda.yaml
@@ -0,0 +1,7 @@
+definitions:
+ - cuda_system_codes:
+ - cuda
+ - cudnn
+
+ - cuda_serial_codes:
+ - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_empty.yaml b/stacks/pinot-noir-gcc/definitions_empty.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ca4ba7fe51ad3279cebc61ff97ee72782db03b81
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_empty.yaml
@@ -0,0 +1 @@
+definitions: []
diff --git a/stacks/pinot-noir-gcc/definitions_helvetios.yaml b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
deleted file mode 100644
index ca4ba7fe51ad3279cebc61ff97ee72782db03b81..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_helvetios.yaml
+++ /dev/null
@@ -1 +0,0 @@
-definitions: []
diff --git a/stacks/pinot-noir-gcc/definitions_helvetios.yaml b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..3552a78f67656b1b6647cced51cbfb18b5882f10
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_helvetios.yaml
@@ -0,0 +1 @@
+definitions_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_izar.yaml b/stacks/pinot-noir-gcc/definitions_izar.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_izar.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
deleted file mode 100644
index 0f156fa830b9567b3e7839218a0a88c16076c6ca..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_kuma_h100.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
deleted file mode 100644
index 0f156fa830b9567b3e7839218a0a88c16076c6ca..0000000000000000000000000000000000000000
--- a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-definitions:
- - cuda_system_codes:
- - cuda
- - cudnn
-
- - cuda_serial_codes:
- - nccl
diff --git a/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..2c65e6a4923cf87b9addc7a71f26c0e6acb0f36f
--- /dev/null
+++ b/stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml
@@ -0,0 +1 @@
+definitions_cuda.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules.yaml b/stacks/pinot-noir-gcc/modules.yaml
index ec5a9a22d3d2419d80a67f1daab7f389858a32b1..5771c991f4e8d758a9a39ff390e80270b3b9cc3a 100644
--- a/stacks/pinot-noir-gcc/modules.yaml
+++ b/stacks/pinot-noir-gcc/modules.yaml
@@ -4,7 +4,7 @@ modules:
lmod:
hash_length: 0
core_compilers:
- - gcc@11.4.1
+ - gcc@11.2.1
hierarchy:
- mpi
all:
@@ -47,7 +47,7 @@ modules:
openmpi:
environment:
set:
- SLURM_MPI_TYPE: pmi2
+ SLURM_MPI_TYPE: pmix
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
append_path:
JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
diff --git a/stacks/pinot-noir-gcc/modules_empty.yaml b/stacks/pinot-noir-gcc/modules_empty.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..21d0544839675931122d4455d2fe32408b425fe5
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_empty.yaml
@@ -0,0 +1,3 @@
+modules:
+ default:
+ lmod: {}
diff --git a/stacks/pinot-noir-gcc/modules_helvetios.yaml b/stacks/pinot-noir-gcc/modules_helvetios.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_helvetios.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_izar.yaml b/stacks/pinot-noir-gcc/modules_izar.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_izar.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_jed.yaml b/stacks/pinot-noir-gcc/modules_jed.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..5be0f314270ddfe2b36d1716f4ea55e43bff43b4
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_jed.yaml
@@ -0,0 +1 @@
+modules_empty.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/modules_kuma_h100.yaml b/stacks/pinot-noir-gcc/modules_kuma_h100.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c4659078af130b7c2532939538c205d6d12edde9
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_kuma_h100.yaml
@@ -0,0 +1,9 @@
+modules:
+ default:
+ lmod:
+ openmpi:
+ environment:
+ set:
+ OMPI_MCA_pml: 'ucx'
+ OMPI_MCA_osc: 'ucx'
+ UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'
diff --git a/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml b/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..7180f05ea42a27f7e59f063ecb69b4d815c4030b
--- /dev/null
+++ b/stacks/pinot-noir-gcc/modules_kuma_l40s.yaml
@@ -0,0 +1 @@
+modules_kuma_h100.yaml
\ No newline at end of file
diff --git a/stacks/pinot-noir-gcc/packages_stack.yaml b/stacks/pinot-noir-gcc/packages_stack.yaml
index c9f989de74c7d3466e7429fce7f815828cd1da59..d12a3aa85defa2721cc41fe5bb6811632456b7d7 100644
--- a/stacks/pinot-noir-gcc/packages_stack.yaml
+++ b/stacks/pinot-noir-gcc/packages_stack.yaml
@@ -9,48 +9,6 @@ packages:
- spec: "^libiconv"
when: "^iconv"
- # - spec: ^intel-oneapi-mpi
- # when: '%oneapi ^mpi'
-
- # - spec: ^openmpi
- # when: '%gcc ^mpi'
-
- # - spec: ^openmpi
- # when: '%aocc ^mpi'
-
- # - spec: ^nvhpc+mpi
- # when: '%nvhpc ^mpi'
-
- # - spec: ^openblas %gcc
- # when: '%gcc ^blas'
-
- # - spec: ^eigen %gcc
- # when: '%gcc ^eigen'
-
- # - spec: ^kokkos %gcc
- # when: '%gcc ^kokkos'
-
- # - spec: ^openblas
- # when: '%gcc ^lapack'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^blas'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^lapack'
-
- # - spec: ^nvhpc+blas
- # when: '%nvhpc ^blas'
-
- # - spec: ^nvhpc+lapack
- # when: '%nvhpc ^lapack'
-
- # - spec: ^openblas
- # when: '%gcc ^py-numpy'
-
- # - spec: ^intel-oneapi-mkl
- # when: '%oneapi ^py-numpy'
-
- spec: '+ipo'
when: '%gcc'
@@ -102,9 +60,12 @@ packages:
prefer: ['~cluster']
kokkos:
- prefer: ['+openmp', '+openmptarget', '+numactl', '+memkind', '+threads',
+ prefer: ['+openmp', '+numactl', '+memkind', '+threads',
'+aggressive_vectorization']
+ kokkos-kernel:
+ prefer: ['+openmp', '+threads']
+
lammps:
require: ['build_type=Release', '+asphere', '+atc', '+body', '+class2',
'+colloid', '+compress', '+coreshell', '+dipole', '+diffraction', '+extra-dump',
@@ -158,6 +119,9 @@ packages:
py-torch:
require: ['+mpi']
+ scotch:
+ prefer: ['~mpi']
+
suite-sparse:
require:
- spec: '+graphblas'
@@ -167,7 +131,7 @@ packages:
ucx:
require:
- - spec: +rdmacm +rc +dc +ud +cma +verbs
+ - spec: '+rdmacm +rc +dc +ud +cma +verbs'
zlib-ng:
require: ['build_system=autotools']
diff --git a/stacks/pinot-noir-gcc/spack.yaml b/stacks/pinot-noir-gcc/spack.yaml
index bc226accf7353bfef40695b18e5928f212eac34f..e355a2ad4b38e3441b26146fa1e87c66fa720eab 100644
--- a/stacks/pinot-noir-gcc/spack.yaml
+++ b/stacks/pinot-noir-gcc/spack.yaml
@@ -5,7 +5,7 @@ spack:
- packages_stack.yaml
- packages_env.yaml
- - modules_stack.yaml
+ - modules_env.yaml
- definitions_env.yaml
@@ -18,21 +18,12 @@ spack:
- mpi_gcc:
- openmpi@5.0.3
- - mpi_oneapi:
- - intel-oneapi-mpi@2021.12.1
-
- blas_gcc:
- openblas@0.3.26
- - blas_oneapi:
- - intel-oneapi-mkl@2024.1.0
-
- python_gcc:
- python@3.11.7
- - python_oneapi:
- - python@3.11.7
-
# -------------------------------------------------------------------------
# Core packages
# -------------------------------------------------------------------------
@@ -74,30 +65,14 @@ spack:
- cuda_serial_codes: []
-
- # -------------------------------------------------------------------------
- # Generic definitions
- # -------------------------------------------------------------------------
- - mpis:
- - $mpi_oneapi
- - $mpi_gcc
-
specs:
- matrix:
- [$compilers_specs]
- - matrix:
- - [$blas_oneapi]
- - [$%compiler_oneapi]
-
- matrix:
- [$blas_gcc]
- [$%compiler_gcc]
- - matrix:
- - [$mpi_oneapi]
- - [$%compiler_oneapi]
-
- matrix:
- [$mpi_gcc]
- [$%compiler_gcc]
@@ -114,14 +89,6 @@ spack:
- [$serial_codes]
- [$%compilers]
- - matrix:
- - [$gcc_serial_codes]
- - [$%compiler_gcc]
-
- - matrix:
- - [$oneapi_serial_codes]
- - [$%compiler_oneapi]
-
- matrix:
- [$cuda_serial_codes]
- [$%compiler_gcc]