Skip to content
Snippets Groups Projects
Commit 8267dab3 authored by Nicolas Richart's avatar Nicolas Richart
Browse files

Merge branch 'feat/stack_pinot_noir' into 'main' 

Feat/stack pinot noir

See merge request !3
parents 9d2ee174 761908ba
No related branches found
No related tags found
1 merge request!3Feat/stack pinot noir
Pipeline #203789 passed
Stage: .pre
Stage: environment
Stage: compilers
Stage: concretize
Stage: install
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 267 additions and 83 deletions
.gitignore
+ 1
− 0
View file @ 8267dab3
.DS_Store
.idea
*.log
spack_*.json
tmp/
.spack-env
compiler_definitions.yaml
.gitlab-ci.yml
+ 31
− 30
View file @ 8267dab3
......@@ -23,28 +23,28 @@ variables:
.parallel_definition:
parallel:
matrix:
# - environment: [jed]
# slurm_options: ["-c 72 -p jed -q jed"]
# stack: ["pinot-noir"]
# app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
# - environment: [helvetios]
# slurm_options: ["-c 36"]
# stack: ["pinot-noir"]
# app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
- environment: [kuma_h100]
# - environment: [izar]
# slurm_options: ['-c 40 --gpus 2']
# apptainer_options: ['-nv']
# stack: ["pinot-noir"]
# - environment: [jed]
# slurm_options: ["-c 72 -p jed -q jed"]
# stack: ["pinot-noir"]
# app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
- environment: [kuma-h100]
slurm_options: ["-c 64"]
stack: ["pinot-noir-gcc"]
stack: ["pinot-noir"]
apptainer_options: ['--nv']
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
- environment: [kuma_l40s]
- environment: [kuma-l40s]
slurm_options: ["-c 64"]
stack: ["pinot-noir-gcc"]
stack: ["pinot-noir"]
apptainer_options: ['--nv']
app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"]
# - environment: [izar]
# slurm_options: ['-c 40 --gpus 2']
# apptainer_options: ['-nv']
# stack: ["pinot-noir"]
tags:
- ${environment}
- stack
......@@ -71,21 +71,19 @@ variables:
FAKEHOME: "/scratch/$(id -un)"
COMMAND_OPTIONS_SBATCH: ${slurm_options}
STACK_VERSION: $(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/config.json)
image_name: "${stack}-${environment}-${SQUASHFS_ID}"
image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
squashfs_image: $(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
APPTAINER_EXEC_OPTIONS: >-
${apptainer_options}
--cleanenv
-H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un)
--env PYTHONUNBUFFERED=1
--bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache
--bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror
--bind ${CI_DATA_ST}/squashfs-cache/:/squashfs-cache
--bind ${CI_DATA_ST}/overlayfs:/overlayfs
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}"
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}"
# after_script:
# - if [ $CI_JOB_STATUS != "success" ]; then exit 0; fi
# - ${CI_PROJECT_DIR}/ci/update_squashfs.sh
.spack_cache:
cache:
......@@ -116,7 +114,7 @@ spack:setup:
stage: environment
extends:
- .parallel_job
- .spack_cache
#- .spack_cache
script:
- ci/setup_spack.sh
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
......@@ -127,11 +125,11 @@ spack:install_compilers:
stage: compilers
extends:
- .parallel_job
- .spack_cache
#- .spack_cache
script:
- ci/install_compilers.sh
- source ci/stack_env.sh
- ${STACK_LOCATION}/spack/bin/spack -e ${environment} config blame compilers
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} config blame compilers
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
artifacts:
reports:
......@@ -144,16 +142,19 @@ spack:concretize:
stage: concretize
extends:
- .parallel_job
- .spack_cache
#- .spack_cache
script:
- source ci/stack_env.sh
- ${STACK_LOCATION}/spack/bin/spack -e ${environment}
config blame | tee config-${environment}-${stack}.log
- ${STACK_LOCATION}/spack/bin/spack -e ${environment}
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
concretize | tee concretize-${environment}-${stack}.log
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment}
mirror create -D -d ${MOUNT_POINT}/spack-mirror -a || /usr//bin/true
- cp ${SPACK_SYSTEM_CONFIG_PATH}/spack.lock spack-${environment}-${stack}.lock
- ${CI_PROJECT_DIR}/ci/update_squashfs.sh
artifacts:
......@@ -169,12 +170,15 @@ spack:install:
stage: install
extends:
- .parallel_job
- .spack_cache
#- .spack_cache
script:
- source ci/stack_env.sh
- echo "{}" > spack-install-${environment}.xml
- ${STACK_LOCATION}/spack/bin/spack -e ${environment} install
# Cleaning patches in case one has a different hash
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} clean -m
- ${STACK_LOCATION}/spack/bin/spack --color always -e ${environment} install
--log-file spack-install-${environment}.xml
--log-format junit
--only-concrete
......@@ -206,15 +210,12 @@ spack:mksquashfs:
stage: deploy
extends:
- .parallel_job
- .spack_cache
#- .spack_cache
script:
- source ci/stack_env.sh
- mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${SQUASHFS_ID}.sqfs
- echo "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
- "mksquashfs ${STACK_LOCATION} stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/stack/${stack}/stack-${stack}-${environment}-${CI_COMMIT_REF_SLUG}.sqfs"'
needs:
- job: spack:install
timeout: 10h
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
#ressource_group: mksquashfs
timeout: 1h
ci/apptainer_by_hand.sh
+ 6
− 3
View file @ 8267dab3
......@@ -6,7 +6,7 @@ srun_options="-N1 -n1 -c 64 -p h100"
apptainer_options="--nv"
export stack=pinot-noir
export environment=kuma_h100
export environment=kuma-h100
export SQUASHFS_ID=local
export CI_PIPELINE_ID=1337
......@@ -20,8 +20,10 @@ export STACK_VERSION=$(jq -Mrc .stack.version ${CI_PROJECT_DIR}/stacks/${stack}/
APPTAINER_IMAGE=~/rhel9-kuma.sif
image_name=${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID}
./ci/prepare_squashfs.sh
squashfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}*.sqfs 2> /dev/null | head -1)
#./ci/prepare_squashfs.sh
srun ${srun_options} --pty apptainer run \
${apptainer_options}\
......@@ -32,11 +34,12 @@ srun ${srun_options} --pty apptainer run \
--bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror \
--bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache \
--bind ${CI_DATA_ST}/overlayfs:/overlayfs \
--env PYTHONUNBUFFERED=1 \
--env stack=${stack} \
--env environment=${environment} \
--env GPG_PRIVATE_KEY=${GPG_PRIVATE_KEY} \
--env CI_PROJECT_DIR=${CI_PROJECT_DIR} \
--env CI_JOB_ID=${CI_JOB_ID} \
--env CI_PIPELINE_ID=${CI_PIPELINE_ID} \
--env SQUASHFS_ID=${SQUASHFS_ID} \
--fusemount "host:${CI_PROJECT_DIR}/ci/squashfuse_ll.sh ${CI_DATA_ST} ${image_name} ${squashfs_image} /overlayfs/lower-${image_name}" \
--fusemount "container:${CI_PROJECT_DIR}/ci/fuse-overlayfs.sh ${image_name} ${MOUNT_POINT}/${stack}/${environment}/${STACK_VERSION}" \
......
ci/debug/spack_lock_info.sh 0 → 100755
+ 133
− 0
View file @ 8267dab3
#!/usr/bin/env sh
get_all_dep(){
file=$2
deps=$(jq --arg h "$1" -Mrc '.concrete_specs | to_entries | map(select(.key | match($h))) | .[] | .value.dependencies[] | "\(.hash)"' $file)
for i in $deps
do
jq --arg h "$i" -c '.concrete_specs
| to_entries
| map(select(.key | match($h)))
| .[]
| {
name: .value.name,
key: .key,
version: .value.version,
compiler: (.value.compiler | "\(.version)"),
target: (.value.arch.target | if type == "object" then .name else . end),
dep: .value.dependencies[]
| {
name: .name,
key: .hash,
type: .parameters.deptypes,
virtuals: .parameters.virtuals
}
}' $file
done
}
get_hash_dep(){
jq --arg h "$1" -c '.concrete_specs
| to_entries
| map(select(.key | match($h)))
| .[]
| {
name: .value.name,
key: .key,
version: .value.version,
compiler: (.value.compiler | "\(.version)"),
target: (.value.arch.target | if type == "object" then .name else . end),
dep: .value.dependencies[]
| {
name: .name,
key: .hash,
type: .parameters.deptypes,
virtuals: .parameters.virtuals
}
}' $2
}
get_name_dep(){
jq --arg n "$1" -c '.concrete_specs
| to_entries
| map(select(.value.name == $n))
| .[]
| {
name: .value.name,
key: .key,
version: .value.version,
compiler: (.value.compiler | "\(.version)"),
target: (.value.arch.target | if type == "object" then .name else . end),
dep: .value.dependencies[]
| {
name: .name,
key: .hash,
type: .parameters.deptypes,
virtuals: .parameters.virtuals
}
}' $2
}
usage() {
echo "-h [hash]"
echo "-n [name]"
echo "-f [file]"
}
OPTSTRING=":h:n:f:a"
all=0
hash=""
name=""
file=""
while getopts ${OPTSTRING} opt; do
case ${opt} in
f)
file=${OPTARG}
;;
h)
hash=${OPTARG}
;;
n)
name=${OPTARG}
;;
a)
all=1
;;
:)
echo "Option -${OPTARG} requires an argument."
usage
exit 1
;;
?)
echo "Invalid option: -${OPTARG}."
usage
exit 1
;;
esac
done
if [ "x$file" == "x" ]
then
echo "No file specified"
usage
exit 1
fi
echo "name ($name), hash ($hash), file($file), all($all)"
if [ "x$hash" != "x" ]
then
if [ $all -eq 1 ]
then
get_all_dep $hash $file
else
get_hash_dep $hash $file
fi
fi
if [ "x$name" != "x" ]
then
get_name_dep $name $file
fi
ci/install_compilers.sh
+ 11
− 7
View file @ 8267dab3
......@@ -73,13 +73,6 @@ for c in "core_compilers" "compilers"; do
fi
done
export target=$(jq -Mrc '
.stack.targets
| to_entries
| .[]
| select(.key == env.environment)
| .value' ${STACK_CONFIG})
compilers_specs=$(jq -Mr '
.stack
| .system_arch as $arch
......@@ -131,10 +124,21 @@ jq -Mrc '
| .system_arch as $arch
| .compilers
| to_entries
| map(select(.value.constraint == null))
| map(" - compiler_\(.key): [\(.value.compiler)@\(.value.version) target=\(env.target)]")
| .[]
' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
jq -Mrc '
.stack
| .system_arch as $arch
| .compilers
| to_entries
| map(select(.value.constraint != null))
| map(" - compiler_\(.key): []\n - when: \(.value.constraint)\n compiler_\(.key): [\(.value.compiler)@\(.value.version) target=\(env.target)]")
| .[]
' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
cat ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml
#yq -y -s ".[0].spack.definitions = .[1].definitions + .[0].spack.definitions | .[0]" ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml ${SPACK_SYSTEM_CONFIG_PATH}/compiler_definitions.yaml > ${SPACK_SYSTEM_CONFIG_PATH}/spack.yaml.new
......
ci/prepare_squashfs.sh
+ 18
− 9
View file @ 8267dab3
......@@ -21,20 +21,29 @@ set +o errexit
sqfs_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${image_name}.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
echo "No MR ($squash_id) squashfs found"
# Check if MR default branch as squashfs
sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
echo "No MR ($squash_id-${CI_PIPELINE_ID}) squashfs found"
# look for base in the branch
sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_id}.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
echo "No default branch ($squash_base) squashfs found"
echo "Creating an empty one"
empty=$(mktemp -d)
sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
mksquashfs ${empty} ${sqfs_base_image}
echo "No MR ($squash_id) squashfs found"
# look for base in default branch
sqfs_base_image=$(ls -t1 ${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}*.sqfs 2> /dev/null | head -1)
if [ $? -ne 0 ]
then
echo "No default branch ($squash_base) squashfs found"
echo "Creating an empty one"
empty=$(mktemp -d)
sqfs_base_image="${CI_DATA_LT}/squashfs-cache/${stack}-${environment}-${squash_base}-initial.sqfs"
mksquashfs ${empty} ${sqfs_base_image}
else
echo "Found ${sqfs_base_image}"
fi
else
echo "Found ${sqfs_base_image}"
fi
set -o errexit
# Link MR sqaushfs to the one of default branch
......
ci/setup_spack.sh
+ 52
− 27
View file @ 8267dab3
......@@ -26,30 +26,48 @@ then
spack/bin/spack env create ${environment}
fi
for file in mirrors packages concretizer config spack definitions modules
do
if [ -e "${STACK_CONFIG_PATH}/${file}.yaml" ]; then
echo "Copying ${STACK_CONFIG_PATH}/${file}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
cp "${STACK_CONFIG_PATH}/${file}.yaml" \
${SPACK_SYSTEM_CONFIG_PATH}
cp "${STACK_CONFIG_PATH}/${file}.yaml" spack/etc/spack
fi
if [ -e "${STACK_CONFIG_PATH}/${file}_stack.yaml" ]; then
echo "Copying ${STACK_CONFIG_PATH}/${file}_stack.yaml to ${SPACK_SYSTEM_CONFIG_PATH}"
cp "${STACK_CONFIG_PATH}/${file}_stack.yaml" \
${SPACK_SYSTEM_CONFIG_PATH}
fi
if [ -e "${STACK_CONFIG_PATH}/${file}_${environment}.yaml" ]; then
echo "Copying ${STACK_CONFIG_PATH}/${file}_${environment}.yaml to ${SPACK_SYSTEM_CONFIG_PATH}/${file}_env.yaml"
cp "${STACK_CONFIG_PATH}/${file}_${environment}.yaml" \
${SPACK_SYSTEM_CONFIG_PATH}/${file}_env.yaml
fi
done
echo "Copying cp ${STACK_CONFIG_PATH}/spack.yaml to ${SPACK_SYSTEM_CONFIG_PATH}/"
cp ${STACK_CONFIG_PATH}/spack.yaml ${SPACK_SYSTEM_CONFIG_PATH}
cp -r "${STACK_CONFIG_PATH}/templates" ${SPACK_SYSTEM_CONFIG_PATH}
declare -A file_mapping
file_mapping["_common"]=""
file_mapping["_stack"]="_stack"
file_mapping["_acc_${acc_type}"]="_accel"
file_mapping["_${environment}"]="_env"
declare -A placeholders
placeholders["{target}"]="${target}"
placeholders["{gcc_version}"]=$(jq -Mrc '.stack.compilers.gcc.version' ${STACK_CONFIG})
if [ "${accelerator}" != "none" ]
then
placeholders["{${acc_type}_arch}"]="${acc_arch}"
fi
echo blip
for file in mirrors packages concretizer config definitions modules
do
for suffix in ${!file_mapping[@]}
do
src=${STACK_CONFIG_PATH}/${file}${suffix}.yaml
dst=${SPACK_SYSTEM_CONFIG_PATH}/${file}"${file_mapping[$suffix]}".yaml
if [ -e "${STACK_CONFIG_PATH}/${file}${suffix}.yaml" ]
then
echo "Copying $src to $dst"
cp "$src" "$dst"
for ph in ${!placeholders[@]}
do
rep=${placeholders[$ph]}
sed -i -e "s/$ph/$rep/g" "$dst"
done
fi
done
done
cat <<EOF > ${SPACK_SYSTEM_CONFIG_PATH}/config_stack.yaml
config:
......@@ -78,7 +96,6 @@ mirrors:
${mirrors}
EOF
cp "${SPACK_SYSTEM_CONFIG_PATH}/mirrors.yaml" spack/etc/spack
echo "Setting up packages"
......@@ -91,16 +108,19 @@ echo "packages:" > ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
export system_compiler=$(jq -Mrc '
.stack
| .system_arch as $arch
| .system_compiler
| to_entries
| .[].value
| "\(.compiler)@\(.version) \($arch)"
| "\(.compiler)@\(.version)"
' ${STACK_CONFIG})
export system_arch=$(jq -Mrc '
.stack.system_arch
' ${STACK_CONFIG})
jq -Mrc '
.stack.system_packages
| map(" \(.):\n require:\n - spec: \"%\(env.system_compiler)\"")
| map(" \(.):\n require:\n - \"%\(env.system_compiler)\"\n - \"\(env.system_arch)\"")
| .[]
' ${STACK_CONFIG} >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
......@@ -111,8 +131,6 @@ then
rm packages.yaml.old
fi
cp "${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml" spack/etc/spack
echo "Setting up extra repos"
EXTERNAL_REPOS=$(jq -r '.spack.repos | keys[]' ${STACK_CONFIG})
......@@ -140,7 +158,14 @@ do
fi
done
cp "${SPACK_SYSTEM_CONFIG_PATH}/repos.yaml" spack/etc/spack
for file in mirrors packages concretizer config modules repos
do
if [ -e "${SPACK_SYSTEM_CONFIG_PATH}/${file}.yaml" ]
then
cp "${SPACK_SYSTEM_CONFIG_PATH}/${file}.yaml" spack/etc/spack
fi
done
echo "Setting up buildcache"
......
ci/stack_env.sh
+ 12
− 0
View file @ 8267dab3
......@@ -14,6 +14,18 @@ export SPACK_SYSTEM_CONFIG_PATH=${STACK_LOCATION}/spack/var/spack/environments/$
#export SPACK_USER_CONFIG_PATH=$(mktemp -p /tmp -d slurm_user_config_XXXXXXX)
export environment_type="local_cluster"
export SPACK_VERSION=$(jq -r .spack.version ${STACK_CONFIG})
env_json=$(jq '.stack.environments | to_entries | .[] | select(.key == env.environment) | .value' ${STACK_CONFIG})
export target=$(echo $env_json | jq -Mrc '.target')
export accelerator=$(echo $env_json | jq -Mrc '.accelerator // "none"')
if [ "$accelerator" != "none" ]
then
export acc_type=$(echo $env_json | jq -Mrc '.accelerator.type')
export acc_arch=$(echo $env_json | jq -Mrc '.accelerator.arch')
fi
echo "STACK_CONFIG_PATH: ${STACK_CONFIG_PATH}"
echo "STACK_CONFIG: ${STACK_CONFIG}"
......
ci/update_squashfs.sh
+ 1
− 1
View file @ 8267dab3
......@@ -8,6 +8,6 @@ source ${CI_PROJECT_DIR}/ci/stack_env.sh
cd /squashfs-cache
mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${squash_id}-$(date +'%Y%m%d_%H%M').sqfs
mksquashfs ${STACK_LOCATION} ${stack}-${environment}-${squash_id}-${CI_PIPELINE_ID}-$(date +'%Y%m%d_%H%M').sqfs
#ln -sf ${stack}-${environment}-${squash_id}-$(date +'%Y%m%d_%H%M').sqfs ${stack}-${environment}-${squash_id}.sqfs
stacks/pinot-noir-gcc/config.json
+ 2
− 2
View file @ 8267dab3
......@@ -37,8 +37,8 @@
"helvetios": "skylake_avx512",
"izar": "cascadelake",
"jed": "icelake",
"kuma_l40s": "zen4",
"kuma_h100": "zen4"
"kuma-l40s": "zen4",
"kuma-h100": "zen4"
},
"compilers": {
"gcc": {
......
stacks/pinot-noir-gcc/definitions_izar.yaml deleted 120000 → 0
+ 0
− 1
View file @ 9d2ee174
definitions_cuda.yaml
\ No newline at end of file
stacks/pinot-noir-gcc/definitions_kuma_h100.yaml deleted 120000 → 0
+ 0
− 1
View file @ 9d2ee174
definitions_cuda.yaml
\ No newline at end of file
stacks/pinot-noir-gcc/definitions_kuma_l40s.yaml deleted 120000 → 0
+ 0
− 1
View file @ 9d2ee174
definitions_cuda.yaml
\ No newline at end of file
stacks/pinot-noir-gcc/mirrors.yaml deleted 100644 → 0
+ 0
− 1
View file @ 9d2ee174
mirrors: {}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment