Merge branch 'main' of gitlab.epfl.ch:SCITAS/software-stack/scitas-software-stacks

.gitlab-ci.yml

@@ -19,15 +19,16 @@ variables:
   TERM: ansi
 
 .parallel_definition:
-  image: registry.c4science.ch/scitas-stack/rhel:9.3
   parallel:
     matrix:
       - environment: [jed]
         slurm_options: ["-c 72 -p jed -q jed"]
         stack: ["pinot-noir"]
+        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
       - environment: [helvetios]
         slurm_options: ["-c 36"]
         stack: ["pinot-noir"]
+        app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
       # - environment: [izar]
       #   slurm_options: ['-c 40 --gpus 2']
       #   apptainer_options: ['-nv']
@@ -45,7 +46,7 @@ variables:
     - stack
   variables:
     environment: ${environment}
-
+  image: ${app_image}
 # ------------------------------------------------------------------------------
 # ------------------------------------------------------------------------------
 .parallel_job:

ci/setup_spack.sh

@@ -25,7 +25,7 @@ mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
 
 EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
 
-for file in mirrors.yaml packages.yaml concretizer.yaml config.yaml spack.yaml
+for file in mirrors.yaml packages.yaml concretizer.yaml config.yaml spack.yaml packages_stack.yaml
 do
   if [ -e "${STACK_CONFIG_PATH}/${file}" ]; then
     echo "Copying ${STACK_CONFIG_PATH}/${file} to ${SPACK_SYSTEM_CONFIG_PATH}"
@@ -85,7 +85,7 @@ spack/bin/spack buildcache keys \
 
 cp ${STACK_CONFIG_PATH}/packages_${environment}.yaml ${SPACK_SYSTEM_CONFIG_PATH}/packages_env.yaml
 
-echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages_system.yaml
+echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 export system_compiler=$(jq -Mrc '
 .stack
  | .system_arch as $arch
@@ -99,4 +99,4 @@ jq -Mrc '
 .stack.system_packages
  | map("  \(.):\n    require:\n    - spec: \"%\(env.system_compiler)\"")
  | .[]
-' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages_system.yaml
+' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml

dockerfiles/rhel9-kuma/Dockerfile

+FROM registry.c4science.ch/scitas-stack/rhel9-base:9.0
+
+#dnf install epel-release
+#dnf group install "Development Tools"
+#dnf install curl findutils gcc-gfortran gnupg2 hostname iproute redhat-lsb-core python3 python3-pip python3-setuptools unzip python3-boto3
+
+# spack dependencies
+RUN yum install -y \
+    gcc-g++ gcc-gfortran \
+    patchelf findutils patch \
+    xz bzip2 file gnupg2 hostname iproute unzip\
+    python3 python3-pip python3-setuptools python3-boto3 \
+    git subversion mercurial \
+    jq \
+    && yum -y clean all && rm -fr /var/cache 
+
+# repos for kuma
+ARG SLURM_REPO=SCITAS_product-slurm_repo-slurm-2311
+ARG MLNX_REPO=SCITAS_product-mlnx_repo-mlnx-ofed-24_01-0_3_3_1-5_14_0-70_30_1-rhel90
+ARG CUDA_REPO=SCITAS_product-cuda_repo-cuda-535_154_05-x86_64-rhel9
+ARG GPFS_REPO=SCITAS_product-gpfs_repo-gpfs-519
+
+# stack dependencies
+RUN yum install -y --enablerepo=$SLURM_REPO \
+    slurm-devel pmix-devel \
+    && yum -y clean all && rm -fr /var/cache 
+
+RUN yum install -y --enablerepo=$MLNX_REPO \
+    libibverbs libibverbs-utils \
+    rdma-core rdma-core-devel \
+    hcoll xpmem kmod-knem \
+    && yum -y clean all && rm -fr /var/cache 
+
+RUN yum install -y --enablerepo=$CUDA_REPO \
+    nvidia-driver-devel \
+    && yum -y clean all && rm -fr /var/cache 
+
+RUN yum install -y --enablerepo=$GPFS_REPO \
+    gpfs.base \
+    && yum -y clean all && rm -fr /var/cache 

dockerfiles/rhel9/Dockerfile

-FROM rhel/ubi9
-
-ARG SPACK_VERSION=v0.21.1
-
-ENV SPACK_DISABLE_LOCAL_CONFIG=true   \
-    SPACK_USER_CACHE_PATH=/tmp/spack-cache
-
-RUN echo "10.95.33.172  foreman1.hpc.epfl.ch" >> /etc/hosts
-RUN curl -sS --insecure \
-    'https://foreman1.hpc.epfl.ch/register?activation_keys=ak-helvetios&organization_id=1&update_packages=false' \
-    -H "Authorization: Bearer ${RHEL_TOKEN}" \
-    | bash
-
-
-#dnf install epel-release
-#dnf group install "Development Tools"
-#dnf install curl findutils gcc-gfortran gnupg2 hostname iproute redhat-lsb-core python3 python3-pip python3-setuptools unzip python3-boto3
-
-RUN yum install gcc-g++ gcc-gfortran \
-    patchelf findutils patch \
-    xz bzip2 file gnupg2 hostname iproute unzip\
-    python3 python3-pip python3-setuptools python3-boto3
-    slurm slurm-libpmi pmix \
-    libibverbs libibverbs-utils \
-    rdma-core rdma-core-devel \
-    git subversion mercurial \
-    jq \
-
-#RUN cd /opt && git clone https://github.com/spack/spack.git -b $SPACK_VERSION

stacks/pinot-noir/config.json

@@ -80,7 +80,8 @@
             "slurm",
             "sqlite",
             "texinfo",
-            "zlib-ng"
+            "zlib-ng",
+            "zstd"
         ]
     }
 }

stacks/pinot-noir/spack.yaml

 spack:
   include:
       - compiler_definitions.yaml
+      - packages_stack.yaml
       - packages_env.yaml
-      - packages_system.yaml
   definitions:
     # -------------------------------------------------------------------------
     # PE definition