Merge branch 'main' of gitlab.epfl.ch:SCITAS/software-stack/scitas-software-stacks

.gitlab-ci.yml

@@ -30,12 +30,12 @@ variables:
       #   stack: ["pinot-noir"]
       #   app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
       - environment: [kuma_h100]
-        slurm_options: ["-c 64"]
+        slurm_options: ["-c 64 --reservation scitas"]
         stack: ["pinot-noir-gcc"]
         apptainer_options: ['--nv']
         app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
       - environment: [kuma_l40s]
-        slurm_options: ["-c 64"]
+        slurm_options: ["-c 64 --reservation scitas"]
         stack: ["pinot-noir-gcc"]
         apptainer_options: ['--nv']
         app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]

stacks/pinot-noir-gcc/config.json

@@ -45,6 +45,11 @@
                 "compiler": "gcc",
                 "spec": "gcc@12.3.0 %gcc@11.2.1",
                 "version": "12.3.0"
+            },
+            "nvhpc": {
+                "compiler": "nvhpc",
+                "spec": "nvhpc +blas +lapack +mpi@24.3 %gcc@11.2.1",
+                "version": "24.3"
             }
         },
         "system_packages": [

stacks/pinot-noir-gcc/definitions_cuda.yaml

@@ -2,7 +2,9 @@ definitions:
   - cuda_system_codes:
       - cuda
       - cudnn
+      - cudnn@9.2.1.18-12
 
   - cuda_serial_codes:
       - nccl
+      - nccl@2.20.5-1
       - nvshmem

stacks/pinot-noir-gcc/modules_kuma_h100.yaml

@@ -7,3 +7,8 @@ modules:
             OMPI_MCA_pml: 'ucx'
             OMPI_MCA_osc: 'ucx'
             UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'
+      nccl:
+        environment:
+          set:
+            NCCL_IB_HCA: 'mlx5_2:1,mlx5_3:1'
+            NCCL_SOCKET_IFNAME: 'bond0'

stacks/pinot-noir-gcc/spack.yaml

@@ -59,6 +59,9 @@ spack:
     - serial_codes:
         - python
 
+    - mpi_codes:
+        - ior
+
     # -------------------------------------------------------------------------
     # Empty to specialize in environments
     # -------------------------------------------------------------------------
@@ -88,12 +91,17 @@ spack:
 
     - matrix:
         - [$serial_codes]
-        - [$%compilers]
+        - [$%compiler_gcc]
 
     - matrix:
         - [$cuda_serial_codes]
         - [$%compiler_gcc]
 
+    - matrix:
+        - [$mpi_codes]
+        - [$^mpi_gcc]
+        - [$%compiler_gcc]
+
   view: false
 #    default:
 #      root: /stack/packages/

stacks/pinot-noir/modules_kuma_h100.yaml

@@ -7,3 +7,8 @@ modules:
             OMPI_MCA_pml: 'ucx'
             OMPI_MCA_osc: 'ucx'
             UCX_NET_DEVICES: 'mlx5_2:1,mlx5_3:1'
+    nccl:
+      environment:
+        set:
+          NCCL_IB_HCA: 'mlx5_2:1,mlx5_3:1'
+          NCCL_SOCKET_IFNAME: 'bond0'