Few changes

ci/install_compilers.sh

@@ -40,7 +40,7 @@ spack/bin/spack buildcache create \
 
 core_compiler=$(jq -Mrc '.stack | .core_compiler' config.json)
 compilers=$(jq -Mrc '.stack | .compilers | to_entries | map("\(.key)@\(.value.version)")' config.json)
-compilers_specs=$(jq -Mrc '.stack | .core_compiler as $core | .compilers | map("\(.spec)@\(.version) %\($core)")' config.json)
+compilers_specs=$(jq -Mrc '.stack | .compilers | map("\(.spec)@\(.version)")' config.json)
 
 cat > compiler_definitions.yaml << EOF
 definitions:
@@ -53,10 +53,12 @@ cat compiler_definitions.yaml
 
 for compiler_hash in $(echo ${hashes}); do
     location=$(spack/bin/spack location -i ${compiler_hash})
-    echo "Checking for compiler in: ${location}"
+    echo "Checking for compiler in locations:"
+    echo "  - ${location}"
+    echo "  - ${location}/compiler/latest"
 
     spack/bin/spack compiler find \
         --scope system \
         ${location} \
-        ${location}/compiler/latest # hack to find intel compilers...
+        ${location}/compiler/latest
 done

ci/setup_spack.sh

@@ -11,36 +11,45 @@ else
   git -C spack pull
 fi
 
-echo "Setting up extra repos"
-mkdir -p /stack/extra_repos/
+echo "Adding spack system config file"
+mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
 
 EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' config.json)
 
+for file in mirrors.yaml repos.yaml packages.yaml concretizer.yaml config.yaml
+do
+    if [ -e ${file} ]; then
+        cp ${file} ${SPACK_SYSTEM_CONFIG_PATH}
+    fi
+done
+
+echo "Setting up extra repos"
+
+spack_external_repos=/stack/external_repos
+mkdir -p ${spack_external_repos}
+
+if [ ! -e ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml ]; then
+    echo "repos:" >> {SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
+fi
+
 for repo in ${EXTERNAL_REPOS}
 do
   repo_branch=$(jq -r ".spack.repos.\"${repo}\".branch" config.json)
-  if [ ! -d /stack/extra_repos/${repo} ]; then
+
+  echo " - ${spack_external_repos}/${repo}"
+
+  if [ ! -d ${spack_external_repos}/${repo} ]; then
     echo "Cloning repo: ${repo}"
     url_branch=$(jq -r ".spack.repos.\"${repo}\".url" config.json)
-    git clone -b ${repo_branch} ${url_branch} /stack/extra_repos/$repo
+    git clone -b ${repo_branch} ${url_branch} ${spack_external_repos}/${repo}
   else
     echo "Update repo: ${repo}"
-    git -C /stack/extra_repos/$repo fetch
-    git -C /stack/extra_repos/$repo checkout ${repo_branch}
-    git -C /stack/extra_repos/$repo pull
+    git -C ${spack_external_repos}/$repo fetch
+    git -C ${spack_external_repos}/$repo checkout ${repo_branch}
+    git -C ${spack_external_repos}/$repo pull
   fi
 done
 
-echo "Adding spack system config file"
-mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-
-for file in mirrors.yaml repos.yaml packages.yaml concretizer.yaml config.yaml
-do
-    if [ -e ${file} ]; then
-        cp ${file} ${SPACK_SYSTEM_CONFIG_PATH}
-    fi
-done
-
 echo "Setting up buildcache"
 
 spack/bin/spack gpg trust \

dockerfiles/rhel9/Dockerfile

@@ -22,7 +22,7 @@ RUN yum install gcc-g++ gcc-gfortran \
     python3 python3-pip python3-setuptools python3-boto3
     slurm slurm-libpmi pmix \
     libibverbs libibverbs-utils \
-    rdma-core \
+    rdma-core rdma-core-devel \
     git subversion mercurial \
     jq \
 

packages.yaml

@@ -4,6 +4,11 @@ packages:
       mpi: [openmpi]
       jpeg: [libjpeg]
 
+    require:
+      - spec: "cuda_arch=70"
+        when: "+cuda"
+      - spec: "+ipo"
+        when: "%gcc"
   # ----------------------------------------------------------------------------
   # System dependencies
   # ----------------------------------------------------------------------------
@@ -23,9 +28,31 @@ packages:
   # ----------------------------------------------------------------------------
   # Requirements
   # ----------------------------------------------------------------------------
+  cmake:
+    require:
+      - spec: "+ownlibs"
+
+  cuda:
+    variants: "+allow-unsupported-compilers"
+
+  hdf5:
+    variants: "+hl +threadsafe +szip +cxx +fortran"
+
   libfabric:
     require:
-      - spec: "fabrics=mlx,tcp,udp,verbs,sockets,psm3,shm"
+      - spec: "fabrics=mlx,tcp,udp,verbs,sockets,shm"
+
+  openmpi:
+    require:
+      - spec: "fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs +thread_multiple"
+      - spec: "cuda_arch=70 ^ucx +cuda"
+        when: "+cuda"
+
+  ucx:
+    require:
+      - spec: "+rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs"
+      - spec: "cuda_arch=70 +gdrcopy"
+        when: "+cuda"
   # ----------------------------------------------------------------------------
   # External find
   # ----------------------------------------------------------------------------

repos.yaml

-repos:
-#  - /stack/extra_repos/scitas-externals
-  - /stack/extra_repos/scitas-spack-packages

spack.yaml

@@ -10,28 +10,20 @@ spack:
       root: /stack/spack/opt/spack
 
   definitions:
-    - when: env['environment'] == 'helvetios'
-      serial_codes_per_cluster:
-        - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs ~cuda ~gdrcopy
-    - when: env['environment'] == 'jed'
-      serial_codes_per_cluster:
-        - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs ~cuda ~gdrcopy
-    - when: env['environment'] == 'izar'
-      serial_codes_per_cluster:
-        - ucx +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs +cuda cuda_arch=70 +gdrcopy
+    - serial_codes_per_cluster:
+        - python
+        - hdf5 ~mpi
 
     - mpi_codes:
         - osu-micro-benchmarks
+        - hdf5 +mpi
 
-    - when: env['environment'] == 'helvetios'
-      mpis:
-        - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs
-    - when: env['environment'] == 'jed'
+    - when: env['environment'] in ['helvetios', 'jed']
       mpis:
-        - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs
+        - openmpi
     - when: env['environment'] == 'izar'
       mpis:
-        - openmpi fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs +cuda cuda_arch=70
+        - openmpi +cuda
 
     - all_serial_codes:
         - $serial_codes_per_cluster