CI boilerplate reduction (#34272)
* CI configuration boilerplate reduction and refactor Configuration: - New notation for list concatenation (prepend/append) - New notation for string concatenation (prepend/append) - Break out configuration files for: ci.yaml, cdash.yaml, view.yaml - Spack CI section refactored to improve self-consistency and composability - Scripts are now lists of lists and/or lists of strings - Job attributes are now listed under precedence ordered list that are composed/merged using Spack config merge rules. - "service-jobs" are identified explicitly rather than as a batch CI: - Consolidate common, platform, and architecture configurations for all CI stacks into composable configuration files - Make padding consistent across all stacks (256) - Merge all package -> runner mappings to be consistent across all stacks Unit Test: - Refactor CI module unit-tests for refactor configuration Docs: - Add docs for new notations in configuration.rst - Rewrite docs on CI pipelines to be consistent with refactored CI workflow * Script verbose environ, dev bootstrap * Port #35409
This commit is contained in:
kwryankrattiger
@@ -1,27 +1,15 @@
|
||||
spack:
|
||||
view: false
|
||||
|
||||
concretizer:
|
||||
reuse: false
|
||||
unify: false
|
||||
|
||||
config:
|
||||
build_jobs: 32
|
||||
install_tree:
|
||||
root: /home/software/spack
|
||||
padded_length: 512
|
||||
projections:
|
||||
all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
|
||||
|
||||
# allow deprecated versions in concretizations
|
||||
# required for zlib
|
||||
deprecated: true
|
||||
|
||||
view: false
|
||||
packages:
|
||||
all:
|
||||
target: [x86_64]
|
||||
tbb:
|
||||
require: 'intel-tbb'
|
||||
|
||||
require: intel-tbb
|
||||
definitions:
|
||||
- gcc_system_packages:
|
||||
- matrix:
|
||||
@@ -62,122 +50,12 @@ spack:
|
||||
- $gcc_spack_built_packages
|
||||
|
||||
mirrors:
|
||||
mirror: 's3://spack-binaries/develop/tutorial'
|
||||
|
||||
gitlab-ci:
|
||||
script:
|
||||
- uname -a || true
|
||||
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
|
||||
- nproc
|
||||
- curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz
|
||||
- printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet
|
||||
- tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
|
||||
- . "./share/spack/setup-env.sh"
|
||||
- spack --version
|
||||
- spack arch
|
||||
- spack compiler find
|
||||
- cd ${SPACK_CONCRETE_ENV_DIR}
|
||||
- spack env activate --without-view .
|
||||
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
|
||||
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
|
||||
- if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi
|
||||
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
|
||||
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
|
||||
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
|
||||
- spack --color=always --backtrace ci rebuild
|
||||
after_script:
|
||||
- cat /proc/loadavg || true
|
||||
|
||||
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
|
||||
match_behavior: first
|
||||
mappings:
|
||||
- match:
|
||||
- cmake
|
||||
- dyninst
|
||||
- gcc
|
||||
- mpich
|
||||
- netlib-lapack
|
||||
- trilinos
|
||||
runner-attributes:
|
||||
tags: ["spack", "large", "x86_64"]
|
||||
variables:
|
||||
CI_JOB_SIZE: large
|
||||
KUBERNETES_CPU_REQUEST: 8000m
|
||||
KUBERNETES_MEMORY_REQUEST: 12G
|
||||
|
||||
- match:
|
||||
- autoconf-archive
|
||||
- boost
|
||||
- hdf5
|
||||
- libtool
|
||||
- libxml2
|
||||
- openblas
|
||||
- openmpi
|
||||
- py-beniget
|
||||
- py-scipy
|
||||
- slurm
|
||||
runner-attributes:
|
||||
tags: ["spack", "medium", "x86_64"]
|
||||
variables:
|
||||
CI_JOB_SIZE: "medium"
|
||||
KUBERNETES_CPU_REQUEST: "2000m"
|
||||
KUBERNETES_MEMORY_REQUEST: "4G"
|
||||
|
||||
- match:
|
||||
- automake
|
||||
- bzip2
|
||||
- expat
|
||||
- findutils
|
||||
- gdbm
|
||||
- json-c
|
||||
- libedit
|
||||
- libevent
|
||||
- libfabric
|
||||
- libffi
|
||||
- libiconv
|
||||
- libidn2
|
||||
- libmd
|
||||
- libpciaccess
|
||||
- lua
|
||||
- meson
|
||||
- pdsh
|
||||
- pkgconf
|
||||
- readline
|
||||
- superlu
|
||||
- tar
|
||||
- util-linux-uuid
|
||||
runner-attributes:
|
||||
tags: ["spack", "small", "x86_64"]
|
||||
variables:
|
||||
CI_JOB_SIZE: "small"
|
||||
KUBERNETES_CPU_REQUEST: "500m"
|
||||
KUBERNETES_MEMORY_REQUEST: "500M"
|
||||
|
||||
- match: ['@:']
|
||||
runner-attributes:
|
||||
tags: ["spack", "x86_64"]
|
||||
variables:
|
||||
CI_JOB_SIZE: default
|
||||
|
||||
broken-specs-url: "s3://spack-binaries/broken-specs"
|
||||
|
||||
service-job-attributes:
|
||||
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
|
||||
before_script:
|
||||
- . "./share/spack/setup-env.sh"
|
||||
- spack --version
|
||||
tags: ["spack", "public", "x86_64"]
|
||||
|
||||
signing-job-attributes:
|
||||
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
|
||||
tags: ["spack", "aws"]
|
||||
script:
|
||||
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
|
||||
- /sign.sh
|
||||
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
|
||||
|
||||
mirror: s3://spack-binaries/develop/tutorial
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
image:
|
||||
name: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02
|
||||
entrypoint: ['']
|
||||
cdash:
|
||||
build-group: Spack Tutorial
|
||||
url: https://cdash.spack.io
|
||||
project: Spack Testing
|
||||
site: Cloud Gitlab Infrastructure
|
||||
|
||||
Reference in New Issue
Block a user