Set the "build_jobs" on concretization/generate for CI (#47660)

* Set the "build_jobs" on concretization/generate for CI

build_jobs also controls the concretization pool size. Set this
in the config section for CI generate.

This config is overwritten by build_job CI using the SPACK_BUILD_JOBS
environment variable. This implicitly will drop the default build
CPU request on all "default" grouped build jobs from (max) 16 to 8.

* Add default allocations for build jobs

* Add common jobs and concretize args to ci generate and rebuild

* CI: Specify parallel concretize and build jobs via argument

* Increase power and cray concretization limits

Lowering limits for these stacks creates timeout

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -145,7 +145,7 @@ default:
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ci generate --check-index-only
+      ci generate --check-index-only -j ${SPACK_CONCRETIZE_JOBS}
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   artifacts:
@@ -155,6 +155,7 @@ default:
       - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/providers"
       - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/tags"
   variables:
+    SPACK_CONCRETIZE_JOBS: 4
     KUBERNETES_CPU_REQUEST: 4000m
     KUBERNETES_MEMORY_REQUEST: 16G
     # avoid moving targets like branches and tags
@@ -408,6 +409,8 @@ e4s-oneapi-build:
   extends: [".linux_power"]
   variables:
     SPACK_CI_STACK_NAME: e4s-power
+    # Override concretization pool for metal runners
+    SPACK_CONCRETIZE_JOBS: 16
 
 e4s-power-generate:
   extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
@@ -861,6 +864,10 @@ aws-pcluster-build-neoverse_v1:
 # Cray definitions
 .generate-cray:
   extends: [ ".generate-common", ".base-job" ]
+  variables:
+    # Override concretization pool for metal runners
+    SPACK_CONCRETIZE_JOBS: 16
+
   before_script:
     - echo $PATH
     - module avail

share/spack/gitlab/cloud_pipelines/configs/ci.yaml

@@ -15,7 +15,6 @@ ci:
       - - touch ${SPACK_USER_CACHE_PATH}/cache/*/*  # bump mtime of cache so it is not invalidated
       - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
         - spack compiler list
-        - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
       - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
         # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
       - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
@@ -24,7 +23,7 @@ ci:
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
       script::
       - - spack config blame mirrors
-        - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+        - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       after_script:
       - - cat /proc/loadavg || true
         - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
@@ -34,6 +33,9 @@ ci:
           ${SPACK_ARTIFACTS_ROOT}/user_data/install_times.json || true
       variables:
         CI_JOB_SIZE: "default"
+        SPACK_BUILD_JOBS: "4"
+        KUBERNETES_CPU_REQUEST: "4000m"
+        KUBERNETES_MEMORY_REQUEST: "16G"
         CI_GPG_KEY_ROOT: /mnt/key
         # SPACK_VERBOSE_SCRIPT: "1"
       id_tokens: