From 316dcc1609ca7c5ae7c3206b04baaa1cc8c9633d Mon Sep 17 00:00:00 2001 From: kwryankrattiger <80296582+kwryankrattiger@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:13:23 -0600 Subject: [PATCH] Set the "build_jobs" on concretization/generate for CI (#47660) * Set the "build_jobs" on concretization/generate for CI build_jobs also controls the concretization pool size. Set this in the config section for CI generate. This config is overwritten by build_job CI using the SPACK_BUILD_JOBS environment variable. This implicitly will drop the default build CPU request on all "default" grouped build jobs from (max) 16 to 8. * Add default allocations for build jobs * Add common jobs and concretize args to ci generate and rebuild * CI: Specify parallel concretize and build jobs via argument * Increase power and cray concretization limits Lowering limits for these stacks creates timeout --- lib/spack/spack/cmd/ci.py | 8 ++++++++ .../spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 9 ++++++++- .../spack/gitlab/cloud_pipelines/configs/ci.yaml | 6 ++++-- share/spack/spack-completion.bash | 4 ++-- share/spack/spack-completion.fish | 16 ++++++++++++++-- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index a7d4f097584..e532ab429e7 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -17,6 +17,7 @@ import spack.ci as spack_ci import spack.cmd import spack.cmd.buildcache as buildcache +import spack.cmd.common.arguments import spack.config as cfg import spack.environment as ev import spack.hash_types as ht @@ -116,6 +117,9 @@ def setup_parser(subparser): ) generate.set_defaults(func=ci_generate) + spack.cmd.common.arguments.add_concretizer_args(generate) + spack.cmd.common.arguments.add_common_arguments(generate, ["jobs"]) + # Rebuild the buildcache index associated with the mirror in the # active, gitlab-enabled environment. index = subparsers.add_parser( @@ -145,6 +149,7 @@ def setup_parser(subparser): help="stop stand-alone tests after the first failure", ) rebuild.set_defaults(func=ci_rebuild) + spack.cmd.common.arguments.add_common_arguments(rebuild, ["jobs"]) # Facilitate reproduction of a failed CI build job reproduce = subparsers.add_parser( @@ -440,6 +445,9 @@ def ci_rebuild(args): if not verify_binaries: install_args.append("--no-check-signature") + if args.jobs: + install_args.append("-j{args.jobs}") + slash_hash = spack_ci.win_quote("/" + job_spec.dag_hash()) # Arguments when installing the root from sources diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 95f98840f07..618585a5c9b 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -145,7 +145,7 @@ default: --config-scope "${SPACK_CI_CONFIG_ROOT}" --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}" --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}" - ci generate --check-index-only + ci generate --check-index-only -j ${SPACK_CONCRETIZE_JOBS} --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" artifacts: @@ -155,6 +155,7 @@ default: - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/providers" - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/tags" variables: + SPACK_CONCRETIZE_JOBS: 4 KUBERNETES_CPU_REQUEST: 4000m KUBERNETES_MEMORY_REQUEST: 16G # avoid moving targets like branches and tags @@ -408,6 +409,8 @@ e4s-oneapi-build: extends: [".linux_power"] variables: SPACK_CI_STACK_NAME: e4s-power + # Override concretization pool for metal runners + SPACK_CONCRETIZE_JOBS: 16 e4s-power-generate: extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"] @@ -861,6 +864,10 @@ aws-pcluster-build-neoverse_v1: # Cray definitions .generate-cray: extends: [ ".generate-common", ".base-job" ] + variables: + # Override concretization pool for metal runners + SPACK_CONCRETIZE_JOBS: 16 + before_script: - echo $PATH - module avail diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml index 10eb7459a71..b3e151e5bd9 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -15,7 +15,6 @@ ci: - - touch ${SPACK_USER_CACHE_PATH}/cache/*/* # bump mtime of cache so it is not invalidated - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR} - spack compiler list - - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k @@ -24,7 +23,7 @@ ci: - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k script:: - - spack config blame mirrors - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) after_script: - - cat /proc/loadavg || true - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true @@ -34,6 +33,9 @@ ci: ${SPACK_ARTIFACTS_ROOT}/user_data/install_times.json || true variables: CI_JOB_SIZE: "default" + SPACK_BUILD_JOBS: "4" + KUBERNETES_CPU_REQUEST: "4000m" + KUBERNETES_MEMORY_REQUEST: "16G" CI_GPG_KEY_ROOT: /mnt/key # SPACK_VERBOSE_SCRIPT: "1" id_tokens: diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index b9b4fd2d7a4..fd5812bac3d 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -693,7 +693,7 @@ _spack_ci() { } _spack_ci_generate() { - SPACK_COMPREPLY="-h --help --output-file --optimize --dependencies --prune-dag --no-prune-dag --check-index-only --artifacts-root" + SPACK_COMPREPLY="-h --help --output-file --optimize --dependencies --prune-dag --no-prune-dag --check-index-only --artifacts-root -U --fresh --reuse --fresh-roots --reuse-deps --deprecated -j --jobs" } _spack_ci_rebuild_index() { @@ -701,7 +701,7 @@ _spack_ci_rebuild_index() { } _spack_ci_rebuild() { - SPACK_COMPREPLY="-h --help -t --tests --fail-fast" + SPACK_COMPREPLY="-h --help -t --tests --fail-fast -j --jobs" } _spack_ci_reproduce_build() { diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 7d1e996c3f0..18e317f0273 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -955,7 +955,7 @@ complete -c spack -n '__fish_spack_using_command ci' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command ci' -s h -l help -d 'show this help message and exit' # spack ci generate -set -g __fish_spack_optspecs_spack_ci_generate h/help output-file= optimize dependencies prune-dag no-prune-dag check-index-only artifacts-root= +set -g __fish_spack_optspecs_spack_ci_generate h/help output-file= optimize dependencies prune-dag no-prune-dag check-index-only artifacts-root= U/fresh reuse fresh-roots deprecated j/jobs= complete -c spack -n '__fish_spack_using_command ci generate' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command ci generate' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command ci generate' -l output-file -r -f -a output_file @@ -972,6 +972,16 @@ complete -c spack -n '__fish_spack_using_command ci generate' -l check-index-onl complete -c spack -n '__fish_spack_using_command ci generate' -l check-index-only -d 'only check spec state from buildcache indices' complete -c spack -n '__fish_spack_using_command ci generate' -l artifacts-root -r -f -a artifacts_root complete -c spack -n '__fish_spack_using_command ci generate' -l artifacts-root -r -d 'path to the root of the artifacts directory' +complete -c spack -n '__fish_spack_using_command ci generate' -s U -l fresh -f -a concretizer_reuse +complete -c spack -n '__fish_spack_using_command ci generate' -s U -l fresh -d 'do not reuse installed deps; build newest configuration' +complete -c spack -n '__fish_spack_using_command ci generate' -l reuse -f -a concretizer_reuse +complete -c spack -n '__fish_spack_using_command ci generate' -l reuse -d 'reuse installed packages/buildcaches when possible' +complete -c spack -n '__fish_spack_using_command ci generate' -l fresh-roots -l reuse-deps -f -a concretizer_reuse +complete -c spack -n '__fish_spack_using_command ci generate' -l fresh-roots -l reuse-deps -d 'concretize with fresh roots and reused dependencies' +complete -c spack -n '__fish_spack_using_command ci generate' -l deprecated -f -a config_deprecated +complete -c spack -n '__fish_spack_using_command ci generate' -l deprecated -d 'allow concretizer to select deprecated versions' +complete -c spack -n '__fish_spack_using_command ci generate' -s j -l jobs -r -f -a jobs +complete -c spack -n '__fish_spack_using_command ci generate' -s j -l jobs -r -d 'explicitly set number of parallel jobs' # spack ci rebuild-index set -g __fish_spack_optspecs_spack_ci_rebuild_index h/help @@ -979,13 +989,15 @@ complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help -d 'show this help message and exit' # spack ci rebuild -set -g __fish_spack_optspecs_spack_ci_rebuild h/help t/tests fail-fast +set -g __fish_spack_optspecs_spack_ci_rebuild h/help t/tests fail-fast j/jobs= complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -f -a tests complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -d 'run stand-alone tests after the build' complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -f -a fail_fast complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -d 'stop stand-alone tests after the first failure' +complete -c spack -n '__fish_spack_using_command ci rebuild' -s j -l jobs -r -f -a jobs +complete -c spack -n '__fish_spack_using_command ci rebuild' -s j -l jobs -r -d 'explicitly set number of parallel jobs' # spack ci reproduce-build set -g __fish_spack_optspecs_spack_ci_reproduce_build h/help runtime= working-dir= s/autostart gpg-file= gpg-url=