diff --git a/ci/install_compilers.sh b/ci/install_compilers.sh
index f28cd6cb79e509a009f91205f0328fc5c382f45e..4fc787fdba0c8ac3a886780aea0ccdce165e9c94 100755
--- a/ci/install_compilers.sh
+++ b/ci/install_compilers.sh
@@ -40,7 +40,7 @@ spack/bin/spack buildcache create \
core_compiler=$(jq -Mrc '.stack | .core_compiler' config.json)
compilers=$(jq -Mrc '.stack | .compilers | to_entries | map("\(.key)@\(.value.version)")' config.json)
-compilers_specs=$(jq -Mrc '.stack | .core_compiler as $core | .compilers | map("\(.spec)@\(.version) %\($core)")' config.json)
+compilers_specs=$(jq -Mrc '.stack | .compilers | map("\(.spec)@\(.version)")' config.json)
cat > compiler_definitions.yaml << EOF
definitions:
@@ -53,10 +53,12 @@ cat compiler_definitions.yaml
for compiler_hash in $(echo ${hashes}); do
location=$(spack/bin/spack location -i ${compiler_hash})
- echo "Checking for compiler in: ${location}"
+ echo "Checking for compiler in locations:"
+ echo " - ${location}"
+ echo " - ${location}/compiler/latest"
spack/bin/spack compiler find \
--scope system \
${location} \
- ${location}/compiler/latest # hack to find intel compilers...
+ ${location}/compiler/latest
done
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index 1a18fbaba79b753f995c9e6a12be8123c1b91819..81f611e9141c124486b24ef371ee05a225c7703c 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -11,36 +11,45 @@ else
git -C spack pull
fi
-echo "Setting up extra repos"
-mkdir -p /stack/extra_repos/
+echo "Adding spack system config file"
+mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' config.json)
+for file in mirrors.yaml repos.yaml packages.yaml concretizer.yaml config.yaml
+do
+ if [ -e ${file} ]; then
+ cp ${file} ${SPACK_SYSTEM_CONFIG_PATH}
+ fi
+done
+
+echo "Setting up extra repos"
+
+spack_external_repos=/stack/external_repos
+mkdir -p ${spack_external_repos}
+
+if [ ! -e ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml ]; then
+ echo "repos:" >> {SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
+fi
+
for repo in ${EXTERNAL_REPOS}
do
repo_branch=$(jq -r ".spack.repos.\"${repo}\".branch" config.json)
- if [ ! -d /stack/extra_repos/${repo} ]; then
+
+ echo " - ${spack_external_repos}/${repo}"
+
+ if [ ! -d ${spack_external_repos}/${repo} ]; then
echo "Cloning repo: ${repo}"
url_branch=$(jq -r ".spack.repos.\"${repo}\".url" config.json)
- git clone -b ${repo_branch} ${url_branch} /stack/extra_repos/$repo
+ git clone -b ${repo_branch} ${url_branch} ${spack_external_repos}/${repo}
else
echo "Update repo: ${repo}"
- git -C /stack/extra_repos/$repo fetch
- git -C /stack/extra_repos/$repo checkout ${repo_branch}
- git -C /stack/extra_repos/$repo pull
+ git -C ${spack_external_repos}/$repo fetch
+ git -C ${spack_external_repos}/$repo checkout ${repo_branch}
+ git -C ${spack_external_repos}/$repo pull
fi
done
-echo "Adding spack system config file"
-mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-
-for file in mirrors.yaml repos.yaml packages.yaml concretizer.yaml config.yaml
-do
- if [ -e ${file} ]; then
- cp ${file} ${SPACK_SYSTEM_CONFIG_PATH}
- fi
-done
-
echo "Setting up buildcache"
spack/bin/spack gpg trust \
diff --git a/dockerfiles/rhel9/Dockerfile b/dockerfiles/rhel9/Dockerfile
index ad69766774fb7f2719cb02f21b20fe1c87106eb7..e8e8254a10889cde3d33be8e5b3f4704a4ff7d2e 100644
--- a/dockerfiles/rhel9/Dockerfile
+++ b/dockerfiles/rhel9/Dockerfile
@@ -22,7 +22,7 @@ RUN yum install gcc-g++ gcc-gfortran \
python3 python3-pip python3-setuptools python3-boto3
slurm slurm-libpmi pmix \
libibverbs libibverbs-utils \
- rdma-core \
+ rdma-core rdma-core-devel \
git subversion mercurial \
jq \
diff --git a/packages.yaml b/packages.yaml
index b79c476b8053ba39a88acc5a72d9ee26ddb711e7..a393ce4758025025835ded2b787f1e9cd343c4bb 100644
--- a/packages.yaml
+++ b/packages.yaml
@@ -4,6 +4,11 @@ packages:
mpi: [openmpi]
jpeg: [libjpeg]
+ require:
+ - spec: "cuda_arch=70"
+ when: "+cuda"
+ - spec: "+ipo"
+ when: "%gcc"
# ----------------------------------------------------------------------------
# System dependencies
# ----------------------------------------------------------------------------
@@ -23,9 +28,31 @@ packages:
# ----------------------------------------------------------------------------
# Requirements
# ----------------------------------------------------------------------------
+ cmake:
+ require:
+ - spec: "+ownlibs"
+
+ cuda:
+ variants: "+allow-unsupported-compilers"
+
+ hdf5:
+ variants: "+hl +threadsafe +szip +cxx +fortran"
+
libfabric:
require:
- - spec: "fabrics=mlx,tcp,udp,verbs,sockets,psm3,shm"
+ - spec: "fabrics=mlx,tcp,udp,verbs,sockets,shm"
+
+ openmpi:
+ require:
+ - spec: "fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs +thread_multiple"
+ - spec: "cuda_arch=70 ^ucx +cuda"
+ when: "+cuda"
+
+ ucx:
+ require:
+ - spec: "+rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs"
+ - spec: "cuda_arch=70 +gdrcopy"
+ when: "+cuda"
# ----------------------------------------------------------------------------
# External find
# ----------------------------------------------------------------------------
diff --git a/repos.yaml b/repos.yaml
deleted file mode 100644
index a55159d47a02041f35871463b454bf49bee9370e..0000000000000000000000000000000000000000
--- a/repos.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-repos:
-# - /stack/extra_repos/scitas-externals
- - /stack/extra_repos/scitas-spack-packages
diff --git a/spack.yaml b/spack.yaml
index b8d49457d79cf1724a0e65bd605301255449f7be..c39ae4dcf6535ddf31ade11269d5281f77284c21 100644
--- a/spack.yaml
+++ b/spack.yaml
@@ -10,28 +10,20 @@ spack:
root: /stack/spack/opt/spack
definitions:
- - when: env['environment'] == 'helvetios'
- serial_codes_per_cluster:
- - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs ~cuda ~gdrcopy
- - when: env['environment'] == 'jed'
- serial_codes_per_cluster:
- - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs ~cuda ~gdrcopy
- - when: env['environment'] == 'izar'
- serial_codes_per_cluster:
- - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs +cuda cuda_arch=70 +gdrcopy
+ - serial_codes_per_cluster:
+ - python
+ - hdf5 ~mpi
- mpi_codes:
- osu-micro-benchmarks
+ - hdf5 +mpi
- - when: env['environment'] == 'helvetios'
- mpis:
- - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs
- - when: env['environment'] == 'jed'
+ - when: env['environment'] in ['helvetios', 'jed']
mpis:
- - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs
+ - openmpi
- when: env['environment'] == 'izar'
mpis:
- - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs +cuda cuda_arch=70
+ - openmpi +cuda
- all_serial_codes:
- $serial_codes_per_cluster