CI boilerplate reduction (#34272)
* CI configuration boilerplate reduction and refactor Configuration: - New notation for list concatenation (prepend/append) - New notation for string concatenation (prepend/append) - Break out configuration files for: ci.yaml, cdash.yaml, view.yaml - Spack CI section refactored to improve self-consistency and composability - Scripts are now lists of lists and/or lists of strings - Job attributes are now listed under precedence ordered list that are composed/merged using Spack config merge rules. - "service-jobs" are identified explicitly rather than as a batch CI: - Consolidate common, platform, and architecture configurations for all CI stacks into composable configuration files - Make padding consistent across all stacks (256) - Merge all package -> runner mappings to be consistent across all stacks Unit Test: - Refactor CI module unit-tests for refactor configuration Docs: - Add docs for new notations in configuration.rst - Rewrite docs on CI pipelines to be consistent with refactored CI workflow * Script verbose environ, dev bootstrap * Port #35409
This commit is contained in:
kwryankrattiger
@@ -1,18 +1,5 @@
|
||||
spack:
|
||||
view: false
|
||||
|
||||
concretizer:
|
||||
reuse: false
|
||||
unify: false
|
||||
|
||||
config:
|
||||
build_jobs: 32
|
||||
install_tree:
|
||||
root: /home/software/spack
|
||||
padded_length: 512
|
||||
projections:
|
||||
all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
|
||||
|
||||
packages:
|
||||
cmake:
|
||||
variants: ~ownlibs
|
||||
@@ -72,142 +59,10 @@ spack:
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" }
|
||||
|
||||
gitlab-ci:
|
||||
image: ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01
|
||||
script:
|
||||
- uname -a || true
|
||||
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
|
||||
- nproc
|
||||
- curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz
|
||||
- printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet
|
||||
- tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
|
||||
- . "./share/spack/setup-env.sh"
|
||||
- spack --version
|
||||
- spack arch
|
||||
- cd ${SPACK_CONCRETE_ENV_DIR}
|
||||
- spack env activate --without-view .
|
||||
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
|
||||
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
|
||||
- if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi
|
||||
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
|
||||
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
|
||||
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
|
||||
- spack --color=always --backtrace ci rebuild
|
||||
after_script:
|
||||
- cat /proc/loadavg || true
|
||||
|
||||
match_behavior: first
|
||||
mappings:
|
||||
- match:
|
||||
- llvm
|
||||
- qt
|
||||
- paraview
|
||||
- visit
|
||||
runner-attributes:
|
||||
tags: [ "spack", "huge", "x86_64" ]
|
||||
variables:
|
||||
CI_JOB_SIZE: huge
|
||||
KUBERNETES_CPU_REQUEST: 11000m
|
||||
KUBERNETES_MEMORY_REQUEST: 42G
|
||||
|
||||
- match:
|
||||
- ecp-data-vis-sdk
|
||||
- mesa
|
||||
- openblas
|
||||
- vtk
|
||||
- vtk-m
|
||||
runner-attributes:
|
||||
tags: [ "spack", "large", "x86_64" ]
|
||||
variables:
|
||||
CI_JOB_SIZE: large
|
||||
KUBERNETES_CPU_REQUEST: 8000m
|
||||
KUBERNETES_MEMORY_REQUEST: 12G
|
||||
|
||||
- match:
|
||||
- adios2
|
||||
- ascent
|
||||
- binutils
|
||||
- blt
|
||||
- boost
|
||||
- conduit
|
||||
- double-conversion
|
||||
- dray
|
||||
- eigen
|
||||
- faodel
|
||||
- hdf5
|
||||
- mfem
|
||||
- nasm
|
||||
- openmpi
|
||||
- pegtl
|
||||
- py-cinemasci
|
||||
- raja
|
||||
- vtk-h
|
||||
runner-attributes:
|
||||
tags: [ "spack", "medium", "x86_64" ]
|
||||
variables:
|
||||
CI_JOB_SIZE: "medium"
|
||||
KUBERNETES_CPU_REQUEST: "2000m"
|
||||
KUBERNETES_MEMORY_REQUEST: "4G"
|
||||
|
||||
- match:
|
||||
- darshan-util
|
||||
- docbook-xml
|
||||
- gdbm
|
||||
- gettext
|
||||
- hwloc
|
||||
- libevent
|
||||
- libmd
|
||||
- libpciaccess
|
||||
- libsigsegv
|
||||
- libunwind
|
||||
- libxml2
|
||||
- libzmq
|
||||
- numactl
|
||||
- openssh
|
||||
- pcre
|
||||
- perl-data-dumper
|
||||
- py-cycler
|
||||
- py-decorator
|
||||
- py-mistune
|
||||
- py-pycparser
|
||||
- py-setuptools
|
||||
- py-wheel
|
||||
- readline
|
||||
- sqlite
|
||||
- tar
|
||||
- util-linux-uuid
|
||||
runner-attributes:
|
||||
tags: [ "spack", "small", "x86_64" ]
|
||||
variables:
|
||||
CI_JOB_SIZE: "small"
|
||||
KUBERNETES_CPU_REQUEST: "500m"
|
||||
KUBERNETES_MEMORY_REQUEST: "500M"
|
||||
|
||||
- match: ['@:']
|
||||
runner-attributes:
|
||||
tags: ["spack", "x86_64"]
|
||||
variables:
|
||||
CI_JOB_SIZE: "default"
|
||||
|
||||
broken-specs-url: "s3://spack-binaries/broken-specs"
|
||||
|
||||
service-job-attributes:
|
||||
image: ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01
|
||||
before_script:
|
||||
- . "./share/spack/setup-env.sh"
|
||||
- spack --version
|
||||
tags: ["spack", "public", "medium", "x86_64"]
|
||||
|
||||
signing-job-attributes:
|
||||
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
|
||||
tags: ["spack", "aws"]
|
||||
script:
|
||||
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
|
||||
- /sign.sh
|
||||
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
image: { "name": "ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01", "entrypoint": [""] }
|
||||
|
||||
cdash:
|
||||
build-group: Data and Vis SDK
|
||||
url: https://cdash.spack.io
|
||||
project: Spack Testing
|
||||
site: Cloud Gitlab Infrastructure
|
||||
build-group:: Data and Vis SDK
|
||||
|
||||
Reference in New Issue
Block a user