diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f34a3b5b6fb2e3ded04ae5f8a1ad5b049d520009..44cb37c47c8b3a0c67a4fb79df4e4d0639c4ccf3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,10 +36,12 @@ variables:
- stack
variables:
COMMAND_OPTIONS_SBATCH: ${slurm_options}
- APPTAINER_EXEC_OPTIONS: ${apptainer_options} --bind ${BUILDCACHE}:/buildcache:rw
+ STACK_CONFIG: ${CI_PROJECT_DIR}/stacks/${stack}/config.json
+ STACK_LOCATION: $(jq -r .stack.mount_point ${STACK_CONFIG})
+ APPTAINER_EXEC_OPTIONS: ${apptainer_options} --bind ${BUILDCACHE}:/buildcache:rw --bind ${CI_PROJECT_DIR}:${STACK_LOCATION}
SPACK_USER_CACHE_PATH: "/tmp/spack_cache_${CI_JOB_ID}"
SPACK_USER_CONFIG_PATH: "/tmp/spack_user_${CI_JOB_ID}"
- SPACK_SYSTEM_CONFIG_PATH: "${CI_PROJECT_DIR}/stack/spack/etc"
+ SPACK_SYSTEM_CONFIG_PATH: "${STACK_LOCATION}/stack/${stack}/environment"
BUILDCACHE: "/work/scitas-ge/richart/ci/buildcache"
.spack_cache:
@@ -72,7 +74,7 @@ spack:install_compilers:
- spack/bin/spack -e stacks/${stack} config blame compilers
after_script:
- if [ ! -e spack-install.xml ]; then
- echo "{}" > spack-install.xml
+ echo "{}" > spack-install.xml
fi
artifacts:
reports:
@@ -88,8 +90,8 @@ spack:concretize:
- .spack_cache
- .parallel_job
script:
- - spack/bin/spack -e stacks/${stack} config blame
- - spack/bin/spack -e stacks/${stack} concretize
+ - spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} config blame
+ - spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} concretize
needs:
- job: spack:install_compilers
artifacts:
@@ -103,19 +105,19 @@ spack:install:
- .spack_cache
- .parallel_job
script:
- - spack/bin/spack -e stacks/${stack} install
+ - spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} install
--log-file spack-install.xml
--log-format junit
--only-concrete
--fail-fast
--show-log-on-error
- - spack/bin/spack -e stacks/${stack} buildcache create
+ - spack/bin/spack -e ${SPACK_SYSTEM_CONFIG_PATH} buildcache create
--update-index
--key EDC904DCE3D2E84E
/buildcache
after_script:
- if [ ! -e spack-install.xml ]; then
- echo "{}" > spack-install.xml
+ echo "{}" > spack-install.xml
fi
needs:
- job: spack:concretize
@@ -130,7 +132,7 @@ spack:mksquashfs:
- .spack_cache
- .parallel_job
script:
- - 'mksquashfs ${CI_PROJECT_DIR} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs'
+ - "mksquashfs ${CI_PROJECT_DIR} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
needs:
- job: spack:install
diff --git a/ci/install_compilers.sh b/ci/install_compilers.sh
index 5103114fb1d16f3976dc2102b72c6cbf9fd8f074..cd19eced0c73226ce1af7374bef847ddeb257aec 100755
--- a/ci/install_compilers.sh
+++ b/ci/install_compilers.sh
@@ -1,66 +1,87 @@
#!/usr/bin/env sh
+set -o errexit
+set -o pipefail
+set -o nounset
+
STACK_CONFIG=stacks/${stack}/config.json
-COMPILER_LIST_SPECS=$(jq -Mr '.stack | .core_compiler as $core | .compilers | map("\(.spec)@\(.spec_version) %\($core)")[]' ${STACK_CONFIG})
+STACK_LOCATION=stack/${stack}
echo "Getting system compiler"
-spack/bin/spack compiler find --scope system /usr
-
-spack/bin/spack spec \
- --install-status \
- --namespaces \
- --long \
- --types \
- ${COMPILER_LIST_SPECS}
-
-spack/bin/spack spec \
- --install-status \
- --namespaces \
- --long \
- --types \
- --json \
- ${COMPILER_LIST_SPECS} > compilers_spec.json
-
-echo "Compilers to install:"
-echo "${COMPILER_LIST_SPECS}"
-
-spack/bin/spack install \
- --log-file spack-install.xml \
- --log-format junit \
- --fail-fast \
- --show-log-on-error \
- ${COMPILER_LIST_SPECS}
-
-hashes=$(jq -Mr '.spec.nodes | map("/\(.hash)") | join(" ")' compilers_spec.json)
-
-echo "Adding packages to buildcache"
-spack/bin/spack buildcache create \
- --update-index \
- --key EDC904DCE3D2E84E \
- /buildcache ${hashes}
-
-core_compiler=$(jq -Mrc '.stack | .core_compiler' ${STACK_CONFIG})
-compilers=$(jq -Mrc '.stack | .compilers | to_entries | map("\(.key)@\(.value.version)")' ${STACK_CONFIG})
-compilers_specs=$(jq -Mrc '.stack | .compilers | map("\(.spec)@\(.spec_version)")' ${STACK_CONFIG})
-
-cat > stacks/${stack}/compiler_definitions.yaml << EOF
-definitions:
- - core_compiler: [ ${core_compiler} ]
- - compilers: ${compilers}
- - compilers_specs: ${compilers_specs}
-EOF
-
-cat compiler_definitions.yaml
-
-for compiler_hash in $(echo ${hashes}); do
- location=$(spack/bin/spack location -i ${compiler_hash})
- echo "Checking for compiler in locations:"
- echo " - ${location}"
- echo " - ${location}/compiler/latest/linux"
-
- spack/bin/spack compiler find \
- --scope system \
- ${location} \
- ${location}/compiler/latest/linux
+${STACK_LOCATION}/spack/bin/spack compiler find --scope system /usr
+
+for c in "core_compilers" "compilers"; do
+ echo "Installing $c"
+ _specs=$(comp=$c jq -Mr '.stack | .system_arch as $arch | with_entries(select(.key==env.comp)) | to_entries | .[].value | map("\(.spec) \($arch)")[]' ${STACK_CONFIG})
+
+ echo "Compilers to install:"
+ echo "${_specs}"
+
+ if [ "x${_specs}" != "x" ]; then
+ ${STACK_LOCATION}/spack/bin/spack spec \
+ --install-status \
+ --namespaces \
+ --long \
+ --types \
+ ${_specs}
+
+ ${STACK_LOCATION}/spack/bin/spack spec \
+ --install-status \
+ --namespaces \
+ --long \
+ --types \
+ --json \
+ ${_specs} > compilers_spec.json
+
+ ${STACK_LOCATION}/spack/bin/spack install \
+ --log-file spack-install.xml \
+ --log-format junit \
+ --fail-fast \
+ --show-log-on-error \
+ ${_specs}
+
+ hashes=$(jq -Mr '.spec.nodes | map("/\(.hash)") | join(" ")' compilers_spec.json)
+
+ echo "Adding packages to buildcache"
+ ${STACK_LOCATION}/spack/bin/spack buildcache create \
+ --update-index \
+ --key EDC904DCE3D2E84E \
+ /buildcache ${hashes}
+
+ for compiler_hash in $(echo ${hashes}); do
+ location=$(${STACK_LOCATION}/spack/bin/spack location -i ${compiler_hash})
+ # echo "Checking for compiler"
+ # echo " - ${location}"
+ # echo " - ${location}/compiler/latest"
+ # echo " - ${location}/compiler/latest/linux"
+ # echo " - ${location}/Linux_x86_64/2024/compilers"
+
+ ${STACK_LOCATION}/spack/bin/spack compiler find \
+ --scope system \
+ ${location} \
+ ${location}/compiler/latest \
+ ${location}/compiler/latest/linux \
+ ${location}/Linux_x86_64/2024/compilers
+ done
+ fi
done
+
+compilers_specs=$(jq -Mr '.stack | .system_arch as $arch | .core_compilers + .compilers | map("\(.spec) \($arch)")' ${STACK_CONFIG})
+system_compiler=$(jq -Mrc '.stack | .system_arch as $arch | .system_compiler | map("\(.compiler)@\(.version) \($arch)")' ${STACK_CONFIG})
+core_compilers=$(jq -Mrc '.stack | .system_compiler + .core_compilers | map("\(.compiler)@\(.version)")' ${STACK_CONFIG})
+compilers=$(jq -Mrc '.stack | .compilers | map("\(.compiler)@\(.version)")' ${STACK_CONFIG})
+
+echo "definitions:" > ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
+set +o nounset
+for c in "compilers_specs" "system_compiler" "core_compilers" "compilers"; do
+ if [ "x${!c}" != "x" ]; then
+ echo " - ${c}: ${!c}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
+ fi
+done
+set -o nounset
+
+cat ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
+yq -y -s ".[0].spack.definitions = .[1].definitions + .[0].spack.definitions | .[0]" ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml > ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml.new
+
+mv ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml.new ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml
diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh
index 6a16f2e0d0fae8b2e5bfd946e2a13ca68db1296b..3e12f9eb10c9c5277cc85dbce8f3a0077f3f8c9f 100755
--- a/ci/setup_spack.sh
+++ b/ci/setup_spack.sh
@@ -1,17 +1,27 @@
#!/usr/bin/env sh
-SPACK_VERSION=$(jq -r .spack.version config.json)
-#STACK_LOCATION=$(jq -r .stack.mount_point config.json)
-STACK_LOCATION=stack/
+set -o errexit
+set -o pipefail
+set -o nounset
+
+STACK_CONFIG_PATH=${PWD}/stacks/${stack}
+STACK_CONFIG=${STACK_CONFIG_PATH}/config.json
+SPACK_VERSION=$(jq -r .spack.version ${STACK_CONFIG})
+
+#STACK_LOCATION=$(jq -r .stack.mount_point ${STACK_CONFIG})
+STACK_LOCATION=${PWD}/stack/${stack}
+
+mkdir -p ${STACK_LOCATION}
+cd ${STACK_LOCATION}
echo "Setting up spack"
if [ ! -d spack/.git ]; then
- git clone https://github.com/spack/spack.git -b $SPACK_VERSION spack
+ git clone https://github.com/spack/spack.git
else
git -C spack fetch
- git -C spack checkout $SPACK_VERSION
- git -C spack pull
+# git -C spack pull
fi
+git -C spack checkout $SPACK_VERSION
#cat > spack/etc/spack/config.yaml << EOF
#config:
@@ -23,34 +33,34 @@ fi
echo "Adding spack system config file"
mkdir -p ${SPACK_SYSTEM_CONFIG_PATH}
-EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' config.json)
+EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
-for file in mirrors.yaml repos.yaml packages.yaml concretizer.yaml config.yaml
+for file in mirrors.yaml packages.yaml concretizer.yaml config.yaml spack.yaml
do
- if [ -e "${file}" ]; then
- cp "${file}" ${SPACK_SYSTEM_CONFIG_PATH}
- fi
+ if [ -e "${STACK_CONFIG_PATH}/${file}" ]; then
+ echo "Copying ${STACK_CONFIG_PATH}/${file} to ${SPACK_SYSTEM_CONFIG_PATH}"
+ cp "${STACK_CONFIG_PATH}/${file}" ${SPACK_SYSTEM_CONFIG_PATH}
+ fi
done
echo "Setting up extra repos"
-spack_external_repos=${STACK_LOCATION}/external_repos
+spack_external_repos=external_repos
mkdir -p ${spack_external_repos}
-if [ ! -e ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml ]; then
- echo "repos:" >> {SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
-fi
+echo "repos:" > ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
for repo in ${EXTERNAL_REPOS}
do
- repo_branch=$(jq -r ".spack.repos.\"${repo}\".branch" config.json)
+ repo_branch=$(jq -r ".spack.repos.\"${repo}\".branch" ${STACK_CONFIG})
echo " - ${spack_external_repos}/${repo}"
-
+ echo " - ${STACK_LOCATION}/${spack_external_repos}/${repo}" >> ${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml
if [ ! -d ${spack_external_repos}/${repo} ]; then
echo "Cloning repo: ${repo}"
- url_branch=$(jq -r ".spack.repos.\"${repo}\".url" config.json)
+ url_branch=$(jq -r ".spack.repos.\"${repo}\".url" ${STACK_CONFIG})
git clone -b ${repo_branch} ${url_branch} ${spack_external_repos}/${repo}
+
else
echo "Update repo: ${repo}"
git -C ${spack_external_repos}/$repo fetch
@@ -69,10 +79,12 @@ if [ ! -d /buildcache/build_cache ]; then
-d /buildcache
fi
-spack/bin/spack mirror add \
- --type binary \
- --scope system \
- buildcache file:///buildcache
+if ! spack/bin/spack mirror list | grep buildcache; then
+ spack/bin/spack mirror add \
+ --type binary \
+ --scope system \
+ buildcache file:///buildcache
+fi
spack/bin/spack buildcache keys \
--install \
diff --git a/stacks/pinot-noir/config.json b/stacks/pinot-noir/config.json
index e6db38e2e27c7063b3de602100b87b7aa2ffb98e..bec44f07702ada6fe44298747ed2c0aa2d32c03c 100644
--- a/stacks/pinot-noir/config.json
+++ b/stacks/pinot-noir/config.json
@@ -1,6 +1,6 @@
{
"spack": {
- "version": "develop",
+ "version": "v0.21.2",
"repos": {
"scitas-externals": {
"url": "https://gitlab.epfl.ch/SCITAS/software-stack/spack-repo-externals.git",
@@ -14,17 +14,31 @@
},
"stack": {
"mount_point": "/stack",
- "core_compiler": "gcc@11.4.1 arch=linux-rhel9-x86_64_v2",
- "compilers": {
- "gcc": {
- "spec": "gcc",
- "spec_version": "12.2.0",
+ "system_arch": "arch=linux-rhel9-x86_64_v2",
+ "system_compiler": {
+ "gcc@11": {
+ "compiler": "gcc",
+ "spec": "gcc@11.4.1",
+ "version": "11.4.1"
+ }
+ },
+ "core_compilers": {
+ "gcc@12": {
+ "compiler": "gcc",
+ "spec": "gcc@12.2.0 %gcc@11.4.1",
"version": "12.2.0"
+ }
+ },
+ "compilers": {
+ "gcc@13": {
+ "compiler": "gcc",
+ "spec": "gcc@13.2.0 %gcc@11.4.1",
+ "version": "13.2.0"
},
"oneapi": {
- "spec": "intel-oneapi-compilers",
- "spec_version": "2023.2.1",
- "version": "2023.2.0"
+ "compiler": "oneapi",
+ "spec": "intel-oneapi-compilers@2024.0.2 %gcc@11.4.1",
+ "version": "2024.0.2"
}
}
}
diff --git a/stacks/pinot-noir/config.yaml b/stacks/pinot-noir/config.yaml
index da38440fa33f15288d42d4fb851919591f73e3fc..2321a56890dae6a1bcf6c2234473e656c9e822d7 100644
--- a/stacks/pinot-noir/config.yaml
+++ b/stacks/pinot-noir/config.yaml
@@ -3,4 +3,4 @@ config:
install_missing_compilers: false
install_tree:
- padded_length: true
+ padded_length: 242
diff --git a/stacks/pinot-noir/packages.yaml b/stacks/pinot-noir/packages.yaml
index c823e13aacc56361baf15b5bd6d2d497ab71b40f..d0934ce62228cbd5c4ef5597945212972c8c928f 100644
--- a/stacks/pinot-noir/packages.yaml
+++ b/stacks/pinot-noir/packages.yaml
@@ -1,112 +1,126 @@
packages:
all:
providers:
- mpi: [openmpi]
+ mpi: [openmpi, intel-oneapi-mpi]
jpeg: [libjpeg]
require:
- - spec: "cuda_arch=70"
- when: "+cuda target=cascadelake"
+ - spec: cuda_arch=70
+ when: +cuda target=cascadelake
- - spec: "cuda_arch=90"
- when: "+cuda target=icelake"
+ - spec: cuda_arch=90
+ when: +cuda target=icelake
+ - spec: cuda_arch=90
+ when: +cuda target=cannonlake
- - spec: "+ipo"
- when: "%gcc"
- # ----------------------------------------------------------------------------
- # System dependencies
- # ----------------------------------------------------------------------------
+ # System dependencies # ----------------------------------------------------------------------------
rdma-core:
- version: [44.1]
- buildable: False
+ buildable: false
externals:
- - spec: rdma-core@44.1
- prefix: /usr
+ - spec: rdma-core@44.1
+ prefix: /usr
slurm:
- version: [23-11-0-1]
- buildable: False
+ buildable: false
+ externals:
+ - spec: slurm@23-11-0-1
+ prefix: /usr
+
+ pmix:
+ buildable: false
externals:
- - spec: slurm@23-11-0-1
- prefix: /usr
+ - spec: pmix@5.0.1
+ prefix: /usr
+
# ----------------------------------------------------------------------------
# Requirements
# ----------------------------------------------------------------------------
cmake:
require:
- - spec: "+ownlibs"
+ - spec: +ownlibs
+
+ cuda:
+ require:
+ - spec: '@12.3.0'
# cuda:
# variants: "+allow-unsupported-compilers"
hdf5:
- variants: "+hl +threadsafe +szip +cxx +fortran"
+ variants: +hl +threadsafe +szip +cxx +fortran
libfabric:
require:
- - spec: "fabrics=mlx,tcp,udp,verbs,sockets,shm"
+ - spec: fabrics=mlx,tcp,udp,verbs,sockets,shm
+
+ intel-oneapi-mpi:
+ require:
+ - any_of: ['%oneapi', '%intel']
openmpi:
require:
- - spec: "fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs +thread_multiple"
- - spec: "cuda_arch=70 ^ucx +cuda"
- when: "+cuda"
+ - any_of: ['%gcc']
+ - spec: fabrics=ofi,ucx,verbs schedulers=slurm ~rsh +gpfs
+ - spec: cuda_arch=70 ^ucx +cuda
+ when: +cuda
ucx:
require:
- - spec: "+rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple +verbs"
- - spec: "cuda_arch=70 +gdrcopy"
- when: "+cuda"
+ - spec: +rdmacm +rc +dc +ud +cma +mlx5_dv +parameter_checking +thread_multiple
+ +verbs
+ - spec: cuda_arch=70 +gdrcopy
+ when: +cuda
- # ----------------------------------------------------------------------------
- # External find
- # ----------------------------------------------------------------------------
binutils:
externals:
- - spec: binutils@2.35.2
- prefix: /usr
+ - spec: binutils@2.35.2
+ prefix: /usr
coreutils:
externals:
- - spec: coreutils@8.32
- prefix: /usr
+ - spec: coreutils@8.32
+ prefix: /usr
findutils:
externals:
- - spec: findutils@4.8.0
- prefix: /usr
+ - spec: findutils@4.8.0
+ prefix: /usr
gawk:
externals:
- - spec: gawk@5.1.0
- prefix: /usr
+ - spec: gawk@5.1.0
+ prefix: /usr
openssh:
externals:
- - spec: openssh@8.7p1
- prefix: /usr
+ - spec: openssh@8.7p1
+ prefix: /usr
openssl:
externals:
- - spec: openssl@3.0.7
- prefix: /usr
+ - spec: openssl@3.0.7
+ prefix: /usr
tar:
externals:
- - spec: tar@1.34
- prefix: /usr
+ - spec: tar@1.34
+ prefix: /usr
curl:
externals:
- - spec: curl@7.76.1+gssapi+nghttp2
- prefix: /usr
+ - spec: curl@7.76.1+gssapi+nghttp2
+ prefix: /usr
git:
externals:
- - spec: git@2.39.3~tcltk
- prefix: /usr
+ - spec: git@2.39.3~tcltk
+ prefix: /usr
gmake:
externals:
- - spec: gmake@4.3
- prefix: /usr
+ - spec: gmake@4.3
+ prefix: /usr
+ groff:
+ externals:
+ - spec: groff@1.22.4
+ prefix: /usr
pkgconf:
externals:
- - spec: pkgconf@1.7.3
- prefix: /usr
+ - spec: pkgconf@1.7.3
+ prefix: /usr
subversion:
externals:
- - spec: subversion@1.14.1
- prefix: /usr
+ - spec: subversion@1.14.1
+ prefix: /usr
diff --git a/stacks/pinot-noir/spack.yaml b/stacks/pinot-noir/spack.yaml
index 862aa732666ba3c3f2cd14630e709d3533995d7b..6da8ddc7620afccabc9de69ce91949cc77217e7c 100644
--- a/stacks/pinot-noir/spack.yaml
+++ b/stacks/pinot-noir/spack.yaml
@@ -1,7 +1,4 @@
spack:
- include:
- - compiler_definitions.yaml
-
definitions:
- serial_codes_per_cluster:
- python
@@ -11,10 +8,13 @@ spack:
- osu-micro-benchmarks
- hdf5 +mpi
- - when: env['environment'] in ['helvetios', 'jed']
+ - mpis:
+ - intel-oneapi-mpi
+
+ - when: env['environment'] in ['helvetios']
mpis:
- openmpi
- - when: env['environment'] in ['izar']
+ - when: env['environment'] in ['izar', 'jed']
mpis:
- openmpi +cuda
@@ -23,19 +23,20 @@ spack:
- $mpis
specs:
- - matrix:
- - [$compilers_specs]
- - [$%core_compiler]
- - matrix:
- - [$all_serial_codes]
- - [$%compilers]
- - matrix:
- - [$mpi_codes]
- - [$^mpis]
- - [$%compilers]
+ - matrix:
+ - [$compilers_specs]
+ - matrix:
+ - [$all_serial_codes]
+ - [$%compilers]
+ - matrix:
+ - [$mpis]
+ - matrix:
+ - [$mpi_codes]
+ - [$^mpis]
+ - [$%compilers]
concretizer:
- unify: when_possible
+ unify: false
duplicates:
strategy: minimal