diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 901e1bd90f15832cc16933b49d59d60b0aac963e..af7d6186d13b68730820b114f63b055c6d65e1e4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -105,9 +105,11 @@ spack:install_compilers:
script:
- ci/install_compilers.sh
- source ci/stack_env.sh
- - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} config blame compilers
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame compilers
after_script:
- - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+ - tar cvzf
+ artifacts_${stack}_${environment}.tgz
+ $stack/spack/var/spack/environments
artifacts:
paths:
- artifacts_${stack}_${environment}.tgz
@@ -129,10 +131,12 @@ spack:concretize:
fi
script:
- source ci/stack_env.sh
- - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} config blame
- - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} concretize --force
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} concretize --force
after_script:
- - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+ - tar cvzf
+ artifacts_${stack}_${environment}.tgz
+ $stack/spack/var/spack/environments
artifacts:
paths:
- artifacts_${stack}_${environment}.tgz
@@ -154,19 +158,21 @@ spack:install:
- source ci/stack_env.sh
- echo "{}" > spack-install-${environment}.xml
- - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} install
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment} install
--log-file spack-install-${environment}.xml
--log-format junit
--only-concrete
--fail-fast
--show-log-on-error
- - ${STACK_LOCATION}/spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} buildcache create
+ - ${STACK_LOCATION}/spack/bin/spack -e ${environment}
--update-index
--key EDC904DCE3D2E84E
/buildcache
after_script:
- - tar cvzf artifacts_${stack}_${environment}.tgz $stack/environment
+ - tar cvzf
+ artifacts_${stack}_${environment}.tgz
+ $stack/spack/var/spack/environments
needs:
- job: spack:concretize
artifacts:
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index 44570f047750cf11fed1a334f2afe56aa3a79ef8..38e0d2a30abc1e886795d3e931abf531935a171a 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -20,10 +20,8 @@ git -C spack checkout $SPACK_VERSION
echo "Adding spack system config file"
-echo mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
+spack/bin/spack env create ${environment}
for file in mirrors.yaml packages.yaml concretizer.yaml config.yaml spack.yaml packages_stack.yaml
do
@@ -33,9 +31,25 @@ do
fi
done
+cp -r "${STACK_CONFIG_PATH}/templates" ${SPACK_SYSTEM_CONFIG_PATH}
+
+if [ ! -e ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml ]
+then
+ echo "config:" >> ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml
+fi
+
+cat <<EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/config.yaml
+ template_dirs:
+ - ${SPACK_SYSTEM_CONFIG_PATH}/templates
+
+ build_stage:
+ - $tempdir/$user/spack-stage
+EOF
+
echo "Setting up extra repos"
-spack_external_repos=external_repos
+EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
+spack_external_repos="${SPACK_SYSTEM_CONFIG_PATH}/external_repos"
mkdir -p ${spack_external_repos}
echo "repos:" > ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
diff --git a/ci/stack_env.sh b/ci/stack_env.sh
index 188a79831e422ac0e4c9bbf4c98b7d991661460a..2fb01cb3c0307a81aacbcf1ac72801c402c75391 100644
--- a/ci/stack_env.sh
+++ b/ci/stack_env.sh
@@ -7,7 +7,7 @@ export SPACK_VERSION=$(jq -r .spack.version ${STACK_CONFIG})
export MOUNT_POINT=$(jq -r .stack.mount_point ${STACK_CONFIG})
export STACK_LOCATION=${MOUNT_POINT}/${stack}
-export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/environment
+export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/spack/var/spack/environments/${environment}
export SPACK_USER_CACHE_PATH=$(mktemp -p /tmp -d slurm_user_cache_XXXXXXX)
export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)
diff --git a/stacks/pinot-noir/modules.yaml b/stacks/pinot-noir/modules.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d09950e55dd1eb0cd88869d57550f7f26ec6465
--- /dev/null
+++ b/stacks/pinot-noir/modules.yaml
@@ -0,0 +1,136 @@
+modules:
+ default:
+ enable: [lmod]
+ lmod:
+ hash_length: 0
+ core_compilers:
+ - gcc@11.4.1
+ hierarchy:
+ - mpi
+ all:
+ environment:
+ set:
+ ${PACKAGE}_ROOT: ${PREFIX}
+ suffixes:
+ +mpi: mpi
+ +openmp: openmp
+ threads=openmp: openmp
+ ^fftw+openmp: openmp
+ hdf5=parallel: h5
+ +cuda: cuda
+ +double-gpu: double-gpu
+ +plumed: plumed
+ +unwind: unwind
+ +debug: dbg
+ +ilp64: int64
+ version_suffix=jl: julia
+
+ blacklist_implicits: true
+ gcc:
+ environment:
+ set:
+ CC: ${PREFIX}/bin/gcc
+ CXX: ${PREFIX}/bin/g++
+ F77: ${PREFIX}/bin/gfortran
+ FC: ${PREFIX}/bin/gfortran
+ F90: ${PREFIX}/bin/gfortran
+ cuda@12.4.0:
+ environment:
+ append_path:
+ JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/cuda"
+
+ openmpi:
+ environment:
+ set:
+ SLURM_MPI_TYPE: pmi2
+ OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
+ append_path:
+ JULIA_LOAD_PATH: ":/ssoft/spack/external/julia/syrah.v1/mpi"
+ openblas threads=pthreads:
+ environment:
+ set:
+ OPENBLAS_NUM_THREADS: '1'
+ hdf5~mpi+cxx:
+ environment:
+ set:
+ HDF5_CC: h5cc
+ HDF5_CXX: h5c++
+ HDF5_FC: h5fc
+ hdf5+mpi~cxx:
+ environment:
+ set:
+ HDF5_CC: h5pcc
+ HDF5_FC: h5pfc
+ intel-oneapi-mpi:
+ environment:
+ set:
+ I_MPI_PMI_LIBRARY: /usr/lib64/libpmi2.so
+ I_MPI_EXTRA_FILESYSTEM: '0'
+ SLURM_MPI_TYPE: pmi2
+ julia:
+ autoload: direct
+ metis:
+ suffixes:
+ ~real64: sp
+ molpro:
+ template: modules/group_restricted.lua
+ plumed:
+ environment:
+ set:
+ PLUMED_KERNEL: ${PREFIX}/lib/libplumedKernel.so
+ unset:
+ - PLUMED_ROOT
+ py-horovod:
+ autoload: direct
+ py-keras:
+ autoload: direct
+ py-tensorflow:
+ autoload: direct
+ py-theano:
+ autoload: direct
+ py-torch:
+ autoload: direct
+ py-torchvision:
+ autoload: direct
+ quantum-espresso:
+ suffixes:
+ hdf5=parallel: hdf5
+ scala:
+ autoload: direct
+ scons:
+ suffixes:
+ '^python@:2.99': py2
+ '^python@3:': py3
+
+ abaqus:
+ template: modules/group_restricted.lua
+ adf:
+ template: modules/group_restricted.lua
+ ams:
+ template: modules/group_restricted.lua
+ comsol:
+ environment:
+ prepend_path:
+ MATLABPATH: ${PREFIX}/mli
+ crystal17:
+ template: modules/group_restricted.lua
+ fdtd:
+ environment:
+ prepend_path:
+ PATH: ${PREFIX}/mpich2/nemesis/bin
+ LD_LIBRARY_PATH: ${PREFIX}/mpich2/nemesis/lib
+ set:
+ SLURM_MPI_TYPE: pmi2
+ gaussian:
+ template: modules/group_restricted.lua
+ spark:
+ autoload: direct
+ environment:
+ prepend_path:
+ PATH: /ssoft/spack/scripts/all/spark
+ ucx:
+ environment:
+ unset:
+ - UCX_ROOT
+ vasp:
+ template: modules/group_restricted.lua
diff --git a/stacks/pinot-noir/templates/modules/group_restricted.lua b/stacks/pinot-noir/templates/modules/group_restricted.lua
new file mode 100644
index 0000000000000000000000000000000000000000..1511796f6bb6dfc84b64c9ec5af200c7d4fe2c12
--- /dev/null
+++ b/stacks/pinot-noir/templates/modules/group_restricted.lua
@@ -0,0 +1,10 @@
+{% extends "modules/modulefile.lua" %}
+{% block footer %}
+-- Access is granted only to specific groups, most likely because the software is licensed
+if not isDir("{{ spec.prefix }}") then
+ LmodError (
+ "You don't have the necessary rights to run \"{{ spec.name }}\".\n\n",
+ "\tPlease write an e-mail to 1234@epfl.ch if you need further information on how to get access to it.\n"
+ )
+end
+{% endblock %}