Merge branch 'main' of gitlab.epfl.ch:SCITAS/software-stack/scitas-software-stacks

.gitlab-ci.yml

@@ -105,9 +105,11 @@ spack:install_compilers:
   script:
     - ci/install_compilers.sh
     - source ci/stack_env.sh
-    - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} config blame compilers
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame compilers
   after_script:
-    - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+    - tar cvzf
+        artifacts_${stack}_${environment}.tgz
+        $stack/spack/var/spack/environments
   artifacts:
     paths:
       - artifacts_${stack}_${environment}.tgz
@@ -129,10 +131,12 @@ spack:concretize:
       fi
   script:
     - source ci/stack_env.sh
-    - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} config blame
-    - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} concretize --force
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force
   after_script:
-    - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+    - tar cvzf
+        artifacts_${stack}_${environment}.tgz
+        $stack/spack/var/spack/environments
   artifacts:
     paths:
       - artifacts_${stack}_${environment}.tgz
@@ -154,19 +158,21 @@ spack:install:
     - source ci/stack_env.sh
     - echo "{}" > spack-install-${environment}.xml
 
-    - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} install
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment} install
       --log-file spack-install-${environment}.xml
       --log-format junit
       --only-concrete
       --fail-fast
       --show-log-on-error
 
-    - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} buildcache create
+    - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
       --update-index
       --key EDC904DCE3D2E84E
       /buildcache
   after_script:
-    - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+    - tar cvzf
+        artifacts_${stack}_${environment}.tgz
+        $stack/spack/var/spack/environments
   needs:
     - job: spack:concretize
   artifacts:

ci/setup_spack.sh

@@ -20,10 +20,8 @@ git -C spack checkout $SPACK_VERSION
 
 
 echo "Adding spack system config file"
-echo mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
 
-EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
+spack/bin/spack env create ${environment}
 
 for file in mirrors.yaml packages.yaml concretizer.yaml config.yaml spack.yaml packages_stack.yaml
 do
@@ -33,9 +31,25 @@ do
   fi
 done
 
+cp -r "${STACK_CONFIG_PATH}/templates" ${SPACK_SYSTEM_CONFIG_PATH}
+
+if [ ! -e ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml ]
+then
+  echo "config:" >> ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml
+fi
+
+cat <<EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml
+  template_dirs:
+    - ${SPACK_SYSTEM_CONFIG_PATH}/templates
+
+  build_stage:
+    - $tempdir/$user/spack-stage
+EOF
+
 echo "Setting up extra repos"
 
-spack_external_repos=external_repos
+EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
+spack_external_repos="${SPACK_SYSTEM_CONFIG_PATH}/external_repos"
 mkdir -p ${spack_external_repos}
 
 echo "repos:" > ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml

ci/stack_env.sh

@@ -7,7 +7,7 @@ export SPACK_VERSION=$(jq -r .spack.version ${STACK_CONFIG})
 export MOUNT_POINT=$(jq -r .stack.mount_point ${STACK_CONFIG})
 export STACK_LOCATION=${MOUNT_POINT}/${stack}
 
-export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/environment
+export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/spack/var/spack/environments/${environment}
 
 export SPACK_USER_CACHE_PATH=$(mktemp -p /tmp -d slurm_user_cache_XXXXXXX)
 export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)

stacks/pinot-noir/modules.yaml

+modules:
+  default:
+    enable: [lmod]
+    lmod:
+      hash_length: 0
+      core_compilers:
+        - gcc@11.4.1
+      hierarchy:
+        - mpi
+      all:
+        environment:
+          set:
+            ${PACKAGE}_ROOT: ${PREFIX}
+        suffixes:
+          +mpi: mpi
+          +openmp: openmp
+          threads=openmp: openmp
+          ^fftw+openmp: openmp
+          hdf5=parallel: h5
+          +cuda: cuda
+          +double-gpu: double-gpu
+          +plumed: plumed
+          +unwind: unwind
+          +debug: dbg
+          +ilp64: int64
+          version_suffix=jl: julia
+
+      blacklist_implicits: true
+      gcc:
+        environment:
+          set:
+            CC: ${PREFIX}/bin/gcc
+            CXX: ${PREFIX}/bin/g++
+            F77: ${PREFIX}/bin/gfortran
+            FC: ${PREFIX}/bin/gfortran
+            F90: ${PREFIX}/bin/gfortran
+      cuda@12.4.0:
+        environment:
+          append_path:
+            JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/cuda"
+
+      openmpi:
+        environment:
+          set:
+            SLURM_MPI_TYPE: pmi2
+            OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
+          append_path:
+            JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
+      openblas threads=pthreads:
+        environment:
+          set:
+            OPENBLAS_NUM_THREADS: '1'
+      hdf5~mpi+cxx:
+        environment:
+          set:
+            HDF5_CC: h5cc
+            HDF5_CXX: h5c++
+            HDF5_FC: h5fc
+      hdf5+mpi~cxx:
+        environment:
+          set:
+            HDF5_CC: h5pcc
+            HDF5_FC: h5pfc
+      intel-oneapi-mpi:
+        environment:
+          set:
+            I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so
+            I_MPI_EXTRA_FILESYSTEM: '0'
+            SLURM_MPI_TYPE: pmi2
+      julia:
+        autoload: direct
+      metis:
+        suffixes:
+          ~real64: sp
+      molpro:
+        template: modules/group_restricted.lua
+      plumed:
+        environment:
+          set:
+            PLUMED_KERNEL: ${PREFIX}/lib/libplumedKernel.so
+          unset:
+          - PLUMED_ROOT
+      py-horovod:
+        autoload: direct
+      py-keras:
+        autoload: direct
+      py-tensorflow:
+        autoload: direct
+      py-theano:
+        autoload: direct
+      py-torch:
+        autoload: direct
+      py-torchvision:
+        autoload: direct
+      quantum-espresso:
+        suffixes:
+          hdf5=parallel: hdf5
+      scala:
+        autoload: direct
+      scons:
+        suffixes:
+          '^python@:2.99': py2
+          '^python@3:': py3
+
+      abaqus:
+        template: modules/group_restricted.lua
+      adf:
+        template: modules/group_restricted.lua
+      ams:
+        template: modules/group_restricted.lua
+      comsol:
+        environment:
+          prepend_path:
+            MATLABPATH: ${PREFIX}/mli
+      crystal17:
+        template: modules/group_restricted.lua
+      fdtd:
+        environment:
+          prepend_path:
+            PATH: ${PREFIX}/mpich2/nemesis/bin
+            LD_LIBRARY_PATH: ${PREFIX}/mpich2/nemesis/lib
+          set:
+            SLURM_MPI_TYPE: pmi2
+      gaussian:
+        template: modules/group_restricted.lua
+      spark:
+        autoload: direct
+        environment:
+          prepend_path:
+            PATH: /ssoft/spack/scripts/all/spark
+      ucx:
+        environment:
+          unset:
+            - UCX_ROOT
+      vasp:
+        template: modules/group_restricted.lua

stacks/pinot-noir/templates/modules/group_restricted.lua

+{% extends "modules/modulefile.lua" %}
+{% block footer %}
+-- Access is granted only to specific groups, most likely because the software is licensed
+if not isDir("{{ spec.prefix }}") then
+    LmodError (
+        "You don't have the necessary rights to run \"{{ spec.name }}\".\n\n",
+        "\tPlease write an e-mail to 1234@epfl.ch if you need further information on how to get access to it.\n"
+    )
+end
+{% endblock %}