zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set build_jobs dynamically in CI to avoid oversubscription (#35996)

Browse Source 
Co-authored-by: Zack Galbreath <zack.galbreath@kitware.com>
Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
This commit is contained in:
Harmen Stoppels
2023-03-13 08:29:58 +01:00
committed by GitHub
parent 31de7ea56c
commit 2107b6bf00
4 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
share/spack/gitlab/cloud_pipelines/configs/ci.yaml
View File

@@ -12,6 +12,7 @@ ci:
- - spack compiler find - - spack compiler find
- cd ${SPACK_CONCRETE_ENV_DIR} - cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view . - spack env activate --without-view .
- if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)

1
share/spack/gitlab/cloud_pipelines/configs/config.yaml
View File

@@ -1,5 +1,4 @@
config: config:
build_jobs: 32
concretizer: clingo concretizer: clingo
db_lock_timeout: 120 db_lock_timeout: 120
install_tree: install_tree:

6
share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
View File

@@ -17,7 +17,8 @@ ci:
tags: [ "spack", "huge" ] tags: [ "spack", "huge" ]
variables: variables:
CI_JOB_SIZE: huge CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m SPACK_BUILD_JOBS: "12"
KUBERNETES_CPU_REQUEST: 12000m
KUBERNETES_MEMORY_REQUEST: 42G KUBERNETES_MEMORY_REQUEST: 42G
- match: - match:
@@ -77,6 +78,7 @@ ci:
tags: [ "spack", "large" ] tags: [ "spack", "large" ]
variables: variables:
CI_JOB_SIZE: large CI_JOB_SIZE: large
SPACK_BUILD_JOBS: "8"
KUBERNETES_CPU_REQUEST: 8000m KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G KUBERNETES_MEMORY_REQUEST: 12G
@@ -170,6 +172,7 @@ ci:
tags: [ "spack", "medium" ] tags: [ "spack", "medium" ]
variables: variables:
CI_JOB_SIZE: "medium" CI_JOB_SIZE: "medium"
SPACK_BUILD_JOBS: "2"
KUBERNETES_CPU_REQUEST: "2000m" KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G" KUBERNETES_MEMORY_REQUEST: "4G"
@@ -286,5 +289,6 @@ ci:
tags: [ "spack", "small" ] tags: [ "spack", "small" ]
variables: variables:
CI_JOB_SIZE: "small" CI_JOB_SIZE: "small"
SPACK_BUILD_JOBS: "1"
KUBERNETES_CPU_REQUEST: "500m" KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M" KUBERNETES_MEMORY_REQUEST: "500M"

1
share/spack/gitlab/cloud_pipelines/configs/linux/ppc64le/ci.yaml
View File

@@ -14,6 +14,7 @@ ci:
- spack arch - spack arch
- cd ${SPACK_CONCRETE_ENV_DIR} - cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view . - spack env activate --without-view .
- if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi