Adapting config to work with nvhpc

ci/setup_spack.sh

@@ -54,7 +54,7 @@ do
   do
     src=${STACK_CONFIG_PATH}/${file}${suffix}.yaml
     dst=${SPACK_SYSTEM_CONFIG_PATH}/${file}"${file_mapping[$suffix]}".yaml
-    if [ -e "${STACK_CONFIG_PATH}/${file}${suffix}.yaml" ]
+    if [ -e "${src}" ]
     then
       echo "Copying $src to $dst"
       cp "$src" "$dst"

stacks/pinot-noir/config.json

@@ -79,8 +79,8 @@
             },
             "nvhpc": {
                 "compiler": "nvhpc",
-                "spec": "nvhpc+mpi+lapack+blas default_cuda=12.4 @24.5 %gcc@11.2.1",
-                "version": "24.5",
+                "spec": "nvhpc+mpi+lapack+blas default_cuda=12.5 @24.7 %gcc@11.2.1",
+                "version": "24.7",
                 "constraint": ["kuma-h100", "kuma-l40s", "izar"]
             }
         },

stacks/pinot-noir/definitions_stack.yaml

@@ -198,23 +198,25 @@ definitions:
       - fftw ++mpi +openmp
       - hdf5@1.14.3 ++mpi
       - hpcg
-      - ior
-      - openfoam-org +metis
+      - netcdf-c
       - osu-micro-benchmarks +graphing
       - parmetis
-      - phylobayesmpi
+      - scorep
       - scotch ++mpi
-      - netcdf-c
+
+  - gcc_mpi_codes:
+      - ior
       - netcdf-fortran
-      - parmetis
-      - scorep
+      - openfoam-org +metis
+      - phylobayesmpi
+      - wrf ~pnetcdf build_type=dm+sm
+
+  - nvhpc_mpi_codes:
+      - boost ++mpi
 
   - mpi_python_codes:
       - py-mpi4py
 
-  - gcc_mpi_codes:
-      - wrf ~pnetcdf build_type=dm+sm
-
   - mpi_blas_codes:
       - arpack-ng ++mpi
       - hypre@2.31.0
@@ -236,31 +238,28 @@ definitions:
       - yambo ++mpi io=iotk,etsf-io
 
   - mpi_blas_python_codes:
-      - boost ++mpi
-      - cgal
-      - iq-tree@1.6.12++mpi
-      - neuron ++mpi +python
       - petsc
-      - plumed
-      - py-petsc4py
       - slepc
       - sirius
 
   - gcc_mpi_blas_python_codes:
-      # - eman2 ++mpi
       - adios2
+      - boost ++mpi
+      - cgal
       - cp2k ++mpi +plumed +openmp smm=blas
-      # - fenics +parmetis +hdf5 +scotch +suite-sparse ~vtk ~trilinos
       - fenics-dolfinx
       - gromacs ++mpi +plumed
+      - iq-tree@1.6.12++mpi
       - lammps
+      - neuron ++mpi +python
       - opencv
       - openfoam +metis
-      - paraview@5.12.0 +shared +python +hdf5  +opengl2
+      - paraview@5.12.0 +shared +python +hdf5  +opengl2 ++mpi
+      - plumed
       - py-fenics-dolfinx
       - py-h5py ++mpi
-      # - py-horovod cannot resolve dependencies with pytorch
       - py-keras
+      - py-petsc4py
       - py-tensorflow
       - py-torch
       - py-torchvision

stacks/pinot-noir/packages_acc_cuda.yaml

@@ -19,20 +19,23 @@ packages:
       when: "%gcc"
     - spec: "%gcc@{gcc_version} target={target}"
       when: "%gcc"
+    - spec: "target={target}"
+      when: "%nvhpc"
     require:
-    - '@12.4.1'
+    - spec: '@12.4.1'
+      when: '%gcc'
+    - spec: '@12.5.1'
+      when: '%nvhpc'
 
   hypre:
-    require:
-    - spec: '+cuda cuda_arch={cuda_arch} +unified-memory'
-      when: '%gcc'
+    require: ['+cuda', 'cuda_arch={cuda_arch}', '+unified-memory']
+
 
   hwloc:
     require: ['+cuda', 'cuda_arch={cuda_arch}', '+nvml']
+
   kokkos:
-    require:
-    - spec: '+cuda +cuda_lambda cuda_arch={cuda_arch} +wrapper ~openmptarget'
-      when: '%gcc'
+    require: ['+cuda', '+cuda_lambda', 'cuda_arch={cuda_arch}', '+wrapper', '~openmptarget']
 
   kokkos-kernels:
     require: ['+cuda', '+cublas', '+cusolver', '+cusparse', 'cuda_arch={cuda_arch}']
@@ -56,9 +59,7 @@ packages:
     require: ['~cuda']
 
   petsc:
-    require:
-    - spec: '+cuda cuda_arch={cuda_arch}'
-      when: '%gcc'
+    require: ['+cuda', 'cuda_arch={cuda_arch}']
 
   py-tensorflow:
     require:
@@ -73,6 +74,17 @@ packages:
     - spec: 'target={target}'
       when: '%gcc@{gcc_version}'
 
+  quantum-espresso:
+    require:
+    - spec: '+cuda +mpigpu'
+      when: '%nvhpc'
+
+  sirius:
+    require: ['+cuda', 'cuda_arch={cuda_arch}']
+
+  slepc:
+    require: ['+cuda', 'cuda_arch={cuda_arch}']
+
   suite-sparse:
     require:
     - spec: '+cuda'

stacks/pinot-noir/packages_stack.yaml

@@ -22,12 +22,44 @@ packages:
 
   boost:
     prefer: ['~mpi']
-    require: ['cxxstd=14', '+icu', '+python', '+numpy', '+atomic', '+chrono',
-    '+container', '+date_time', '+filesystem', '+graph', '+iostreams', '~json',
-    '+locale', '+log', '+math', '~pic', '+program_options', '+random', '+regex',
-    '+serialization', '+shared', '+signals', '~singlethreaded', '~stacktrace',
-    '+system', '~taggedlayout', '+test', '+thread', '+timer', '~type_erasure',
-    '~versionedlayout', '+wave', '+exception']
+    require:
+    - 'cxxstd=14'
+    - '+icu'
+    - '+atomic'
+    - '+chrono'
+    - '+container'
+    - '+date_time'
+    - '+filesystem'
+    - '+graph'
+    - '+iostreams'
+    - '~json'
+    - '+locale'
+    - '+log'
+    - '+math'
+    - '~pic'
+    - '+program_options'
+    - '+random'
+    - '+regex'
+    - '+serialization'
+    - '+shared'
+    - '+signals'
+    - '~singlethreaded'
+    - '~stacktrace'
+    - '+system'
+    - '~taggedlayout'
+    - '+test'
+    - '+thread'
+    - '+timer'
+    - '~type_erasure'
+    - '~versionedlayout'
+    - '+wave'
+    - '+exception'
+
+    - spec: '+python +numpy'
+      when: '%gcc'
+
+    - spec: '+python ~numpy'
+      when: '%nvhpc'
 
   cairo:
     # ~X avoids a dependency to python (only prefer due to gtkplus)
@@ -91,9 +123,6 @@ packages:
     prefer:
     - spec: '@2.31.0 ~magma'
 
-  icu4c:
-    require: ['%gcc']
-
   intel-oneapi-mkl:
     prefer: ['~cluster']
 
@@ -201,8 +230,8 @@ packages:
   netcdf-c:
     prefer: ['+mpi']
 
-  nvhpc:
-    require: ['%gcc@11.2.1', 'target=x86_64_v3']
+  opari2:
+    require: ['%gcc']
 
   openblas:
     require: ['build_system=makefile']
@@ -222,17 +251,26 @@ packages:
   openssl:
     prefer: ['+shared', 'certs=system', '~docs']
 
+  otf2:
+    require: ['%gcc']
+
   pango:
-    prefer: [~X]
+    prefer: ['~X']
 
   papi:
-    require: ['+powercap', '+rapl']
+    require:
+    - '+powercap'
+    - '+rapl'
+    - any_of: ['%gcc', '%oneapi']
+
 
   petsc:
-    require: ['+hypre', '+mumps', '+saws', '+scalapack', '+mpi', '+suite-sparse', '+kokkos', '+openmp']
+    require: ['+hypre', '+mumps', '+saws', '+scalapack', '+mpi',
+              '+suite-sparse', '+kokkos', '+openmp', '+superlu-dist']
 
   python:
     require:
+    - '@3.11.7'
     - '+ssl'
     - '+tkinter'
     - '+crypt'
@@ -313,17 +351,13 @@ packages:
   rust:
     require: ['+dev']
 
-  scalapack:
-    require:
-    - spec: 'netlib-scalapack'
-      when: '%gcc'
-
   scotch:
     prefer: ['~mpi']
     require: ['build_system=cmake']
 
   sirius:
     prefer: ['+fortran', '~elpa', '+wannier90']
+    require: ['+openmp']
 
   slepc:
     require:

stacks/pinot-noir/spack.yaml

@@ -15,6 +15,19 @@ spack:
 
       - config_stack.yaml
 
+  packages:
+    blas:
+      require: ['openblas']
+
+    lapack:
+      require: ['openblas']
+
+    mpi:
+      require: ['openmpi']
+
+    scalapack:
+      require: ['netlib-scalapack']
+
   definitions:
     # -------------------------------------------------------------------------
     # PE definition

stacks/pinot-noir/spack_nvhpc.yaml

@@ -15,20 +15,39 @@ spack:
 
       - config_stack.yaml
 
+  packages:
+    blas:
+      require: ['nvhpc +blas']
+
+    lapack:
+      require: ['nvhpc +lapack']
+
+    mpi:
+      require: ['nvhpc +mpi']
+
+    scalapack:
+      require: ['netlib-scalapack']
+
+    nvhpc:
+      require: ['@24.7', '+mpi', '+blas', '+lapack', 'default_cuda=12.5',
+                '%gcc@11.2.1', 'target=x86_64_v3']
+
+    icu4c:
+      require: ['%gcc']
+
   definitions:
     # -------------------------------------------------------------------------
     # PE definition
     # -------------------------------------------------------------------------
     - mpi_nvhpc:
-        - "[virtuals=mpi] nvhpc+mpi+blas+lapack default_cuda=12.4"
+        - "[virtuals=mpi] nvhpc+mpi"
 
     - blas_nvhpc:
-        - "[virtuals=blas,lapack] nvhpc+mpi+blas+lapack default_cuda=12.4"
+        - "[virtuals=blas,lapack] nvhpc +blas +lapack"
 
     - python_nvhpc:
         - python@3.11.7 %gcc@11.2.1
 
-
   specs:
     - matrix:
         - [$compilers_specs]
@@ -38,6 +57,16 @@ spack:
         - [$serial_codes]
         - [$%compiler_nvhpc]
 
+    - matrix:
+        - [$mpi_codes]
+        - [$^mpi_nvhpc]
+        - [$%compiler_nvhpc]
+
+    - matrix:
+        - [$nvhpc_mpi_codes]
+        - [$^mpi_nvhpc]
+        - [$%compiler_nvhpc]
+
     - matrix:
         - [$mpi_blas_codes]
         - [$^blas_nvhpc]
@@ -46,7 +75,6 @@ spack:
 
     - matrix:
         - [$mpi_blas_python_codes]
-        - [$^python_nvhpc]
         - [$^blas_nvhpc]
         - [$^mpi_nvhpc]
         - [$%compiler_nvhpc]