zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set resource requests on package builds (#29922)

Browse Source 
gitlab ci: Set resource requests explicitly

This PR sets resource requests for the Kubernetes executor, which should aid in
better workload scheduling in the cluster.  The specific values were derived from
profile data taken from several full "from scratch" rebuilds in a separate worker pool.

Co-authored-by: Zack Galbreath <zack.galbreath@kitware.com>
This commit is contained in:
Christopher Kotfila 2022-04-19 12:09:32 -04:00 committed by GitHub
parent f0ceeac0a8
commit 4067a28182
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 466 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
View File

@ -37,11 +37,64 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild - spack -d ci rebuild
image:
name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
entrypoint: [ "" ]
mappings: mappings:
- match: [ 'os=ubuntu18.04' ] - match:
- cmake
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [ "" ] }
tags: [ "spack", "public", "large", "x86_64"] tags: [ "spack", "public", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G
- match:
- curl
- gettext
- mpich
- openjpeg
- sqlite
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
- match:
- bzip2
- diffutils
- findutils
- libffi
- libidn2
- libmd
- libsigsegv
- libxml2
- lz4
- openssl
- pkgconf
- tut
- util-linux-uuid
- util-macros
- xz
- zlib
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"
- match:
- 'os=ubuntu18.04'
runner-attributes:
tags: ["spack", "public", "x86_64"]
variables:
CI_JOB_SIZE: "default"
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs" broken-specs-url: "s3://spack-binaries-develop/broken-specs"
@ -50,8 +103,7 @@ spack:
- . "./share/spack/setup-env.sh" - . "./share/spack/setup-env.sh"
- spack --version - spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "x86_64"]
cdash: cdash:
build-group: Build tests for different build systems build-group: Build tests for different build systems
url: https://cdash.spack.io url: https://cdash.spack.io

92
share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
View File

@ -51,15 +51,97 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild - spack -d ci rebuild
mappings: mappings:
- match: [llvm] - match:
- llvm
runner-attributes: runner-attributes:
tags: [ "spack", "public", "huge", "x86_64" ] tags: [ "spack", "public", "huge", "x86_64" ]
- match: [vtk-h, vtk-m, paraview, vtk] variables:
runner-attributes: CI_JOB_SIZE: huge
tags: ["spack", "public", "xlarge", "x86_64"] KUBERNETES_CPU_REQUEST: 11000m
- match: ['@:'] KUBERNETES_MEMORY_REQUEST: 42G
- match:
- ecp-data-vis-sdk
- mesa
- openblas
- paraview
- vtk-m
runner-attributes: runner-attributes:
tags: [ "spack", "public", "large", "x86_64" ] tags: [ "spack", "public", "large", "x86_64" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G
- match:
- adios2
- ascent
- binutils
- blt
- boost
- conduit
- double-conversion
- dray
- eigen
- faodel
- hdf5
- mfem
- nasm
- openmpi
- pegtl
- py-cinemasci
- raja
- vtk-h
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
- match:
- darshan-util
- docbook-xml
- gdbm
- gettext
- hwloc
- libevent
- libmd
- libpciaccess
- libsigsegv
- libunwind
- libxml2
- libzmq
- numactl
- openssh
- pcre
- perl-data-dumper
- py-cycler
- py-decorator
- py-mistune
- py-pycparser
- py-setuptools
- py-wheel
- readline
- sqlite
- tar
- util-linux-uuid
runner-attributes:
tags: [ "spack", "public", "small", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"
- match: ['@:']
runner-attributes:
tags: ["spack", "public", "x86_64"]
variables:
CI_JOB_SIZE: "default"
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs" broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes: service-job-attributes:

167
share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
View File

@ -232,34 +232,177 @@ spack:
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
mappings: mappings:
- match: - match:
- llvm - llvm
- llvm-amdgpu
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: [ "spack", "public", "huge", "x86_64" ] tags: [ "spack", "public", "huge", "x86_64" ]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G
- match: - match:
- cuda - cuda
- dyninst - dyninst
- ginkgo
- hpx - hpx
- kokkos-kernels - kokkos-kernels
- kokkos-nvcc-wrapper
- magma
- mfem
- mpich
- openturns
- precice - precice
- rocblas - raja
- rocsolver - rust
- strumpack - slate
- sundials
- trilinos - trilinos
- vtk-h
- vtk-m - vtk-m
- warpx - warpx
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: [ "spack", "public", "large", "x86_64" ]
tags: ["spack", "public", "xlarge", "x86_64"] variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G
- match:
- adios2
- amrex
- archer
- ascent
- axom
- binutils
- blaspp
- boost
- butterflypack
- cabana
- caliper
- camp
- chai
- conduit
- datatransferkit
- faodel
- ffmpeg
- fftw
- fortrilinos
- gperftools
- gptune
- hdf5
- heffte
- hpctoolkit
- hwloc
- hypre
- kokkos
- lammps
- lapackpp
- legion
- libzmq
- llvm-openmp-ompt
- mbedtls
- netlib-scalapack
- omega-h
- openmpi
- openpmd-api
- pagmo2
- papyrus
- parsec
- pdt
- petsc
- pumi
- py-ipython-genutils
- py-petsc4py
- py-scipy
- py-statsmodels
- py-warlock
- py-warpx
- pygmo
- slepc
- slurm
- strumpack
- sundials
- superlu-dist
- tasmanian
- tau
- upcxx
- vtk-h
- zfp
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
- match:
- alsa-lib
- ant
- antlr
- argobots
- automake
- berkeley-db
- bison
- blt
- cmake
- curl
- darshan-util
- diffutils
- exmcutils
- expat
- flit
- freetype
- gdbm
- gotcha
- hpcviewer
- jansson
- json-c
- libbsd
- libevent
- libjpeg-turbo
- libnrm
- libpng
- libunistring
- lua-luaposix
- m4
- mpfr
- ncurses
- openblas
- openjdk
- papi
- parallel-netcdf
- pcre2
- perl-data-dumper
- pkgconf
- py-alembic
- py-idna
- py-testpath
- qhull
- snappy
- swig
- tar
- tcl
- texinfo
- unzip
- util-linux-uuid
- util-macros
- yaml-cpp
- zlib
- zstd
runner-attributes:
tags: [ "spack", "public", "small", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"
- match: ['os=ubuntu18.04'] - match: ['os=ubuntu18.04']
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"]
tags: ["spack", "public", "large", "x86_64"] variables:
CI_JOB_SIZE: "default"
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs" broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes: service-job-attributes:
@ -267,7 +410,7 @@ spack:
- . "./share/spack/setup-env.sh" - . "./share/spack/setup-env.sh"
- spack --version - spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "x86_64"]
cdash: cdash:
build-group: New PR testing workflow build-group: New PR testing workflow

94
share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
View File

@ -68,19 +68,103 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild - spack -d ci rebuild
mappings: mappings:
- match: [ascent, axom, sundials, umpire, vtk-h, vtk-m] - match:
runner-attributes: - lbann
tags: ["spack", "public", "xlarge", "x86_64"] - openblas
- match: ['os=ubuntu18.04'] - rust
runner-attributes: runner-attributes:
tags: ["spack", "public", "large", "x86_64"] tags: ["spack", "public", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G
- match:
- ascent
- axom
- conduit
- hdf5
- hydrogen
- mfem
- mvapich2
- py-packaging
- py-scipy
- samrai
- vtk-h
- vtk-m
runner-attributes:
tags: ["spack", "public", "medium", "x86_64"]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
- match:
- autoconf-archive
- camp
- cmake
- curl
- czmq
- diffutils
- gawk
- gdbm
- gettext
- glib
- gmake
- libgcrypt
- libpciaccess
- libpng
- libsigsegv
- libsodium
- libxml2
- libyaml
- libzmq
- lua
- lz4
- m4
- meson
- metis
- ninja
- pcre
- pdsh
- perl
- pkgconf
- py-cffi
- py-jsonschema
- py-kiwisolver
- py-pycparser
- py-setuptools-scm
- py-six
- py-wheel
- qhull
- readline
- sed
- slurm
- sqlite
- unzip
- util-linux-uuid
- util-macros
- zfp
- zlib
runner-attributes:
tags: ["spack", "public", "small", "x86_64"]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"
- match: ['os=ubuntu18.04']
runner-attributes:
tags: ["spack", "public", "x86_64"]
variables:
CI_JOB_SIZE: "default"
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
service-job-attributes: service-job-attributes:
before_script: before_script:
- . "./share/spack/setup-env.sh" - . "./share/spack/setup-env.sh"
- spack --version - spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "x86_64"]
cdash: cdash:
build-group: RADIUSS build-group: RADIUSS

76
share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
View File

@ -64,19 +64,76 @@ spack:
- spack env activate --without-view . - spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild - spack -d ci rebuild
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
mappings: mappings:
- match: [llvm] - match:
- cmake
- dyninst
- gcc
- mpich
- netlib-lapack
- trilinos
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } tags: ["spack", "public", "large", "x86_64"]
tags: ["spack", "public", "huge", "x86_64"] variables:
- match: [trilinos, gcc] CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G
- match:
- autoconf-archive
- boost
- hdf5
- libtool
- libxml2
- openblas
- openmpi
- py-beniget
- py-scipy
- slurm
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } tags: ["spack", "public", "medium", "x86_64"]
tags: ["spack", "public", "xlarge", "x86_64"] variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
- match:
- automake
- bzip2
- expat
- findutils
- gdbm
- json-c
- libedit
- libevent
- libfabric
- libffi
- libiconv
- libidn2
- libmd
- libpciaccess
- lua
- meson
- pdsh
- pkgconf
- readline
- superlu
- tar
- util-linux-uuid
runner-attributes:
tags: ["spack", "public", "small", "x86_64"]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"
- match: ['@:'] - match: ['@:']
runner-attributes: runner-attributes:
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } tags: ["spack", "public", "x86_64"]
tags: ["spack", "public", "large", "x86_64"] variables:
CI_JOB_SIZE: default
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs" broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes: service-job-attributes:
@ -84,7 +141,8 @@ spack:
before_script: before_script:
- . "./share/spack/setup-env.sh" - . "./share/spack/setup-env.sh"
- spack --version - spack --version
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "x86_64"]
cdash: cdash:
build-group: Spack Tutorial build-group: Spack Tutorial