gitlab ci: Rework how mirrors are configured (#39939)

Improve how mirrors are used in gitlab ci, where we have until now thought of them as only a string. By configuring ci mirrors ahead of time using the proposed mirror templates, and by taking advantage of the expressiveness that spack now has for mirrors, this PR will allow us to easily switch the protocol/url we use for fetching binary dependencies. This change also deprecates some gitlab functionality and marks it for removal in Spack 0.23: - arguments to "spack ci generate": * --buildcache-destination * --copy-to - gitlab configuration options: * enable-artifacts-buildcache * temporary-storage-url-prefix
2023-10-19 10:04:59 -06:00
parent b2d3e01fe6
commit 46c1a8e4c6
36 changed files with 273 additions and 150 deletions
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -3,6 +3,12 @@ stages: [ "generate", "build", "publish" ]
 variables:
  SPACK_DISABLE_LOCAL_CONFIG: "1"
  SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
+  # PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
+  PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
+  PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
+  # PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
+  PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
+  PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"

 default:
  image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -68,7 +74,9 @@ default:
 ########################################
 .base-job:
  variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
+    # TODO: We can remove this when we drop the "deprecated" stack
+    PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"

  rules:
    - if: $CI_COMMIT_REF_NAME == "develop"
@@ -76,7 +84,7 @@ default:
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
        SPACK_REQUIRE_SIGNING: "True"
        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -86,7 +94,7 @@ default:
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
        SPACK_PRUNE_UNTOUCHED: "False"
        SPACK_PRUNE_UP_TO_DATE: "False"
        SPACK_REQUIRE_SIGNING: "True"
@@ -98,8 +106,8 @@ default:
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_copy_only"
-        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
+        PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
@@ -108,9 +116,16 @@ default:
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_pull_request"
-        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        # TODO: We can remove this when we drop the "deprecated" stack
+        PUSH_BUILDCACHE_DEPRECATED: "${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
        SPACK_PRUNE_UNTOUCHED: "True"
        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+        # TODO: Change sync script to include target in branch name.  Then we could
+        # TODO: have multiple types of "PR" pipeline here.  It would be better if we could
+        # TODO: keep just this one and use a regex to capture the target branch, but so
+        # TODO: far gitlab doesn't support that.
+        PR_TARGET_REF_NAME: "develop"
+        PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
        AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
@@ -126,13 +141,15 @@ default:
    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
    - spack env activate --without-view .
    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
+      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
    - spack
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      ${CI_STACK_CONFIG_SCOPES}
      ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
  after_script:
@@ -182,7 +199,7 @@ default:
    - spack env activate --without-view .
    - spack
      ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --buildcache-destination "${PUSH_BUILDCACHE_DEPRECATED}"
      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
  after_script:
@@ -219,8 +236,7 @@ protected-publish:
    max: 2
    when: ["runner_system_failure", "stuck_or_timeout_failure"]
  variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+    SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
    SPACK_PIPELINE_TYPE: "spack_protected_branch"
    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -253,11 +269,6 @@ protected-publish:
 # you should inlclude your custom definitions at the end of the of the
 # extends list.
 #
-# Also note that if extending .base-job, the mirror url given in your
-# spack.yaml should take the form:
-#
-#     s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
-#
 ########################################
 # My Super Cool Pipeline
 ########################################
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -21,7 +21,8 @@ ci:
        - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
        - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
      script::
-      - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - - spack config blame mirrors
+        - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
      - - spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
          --prefix /home/software/spack:${CI_PROJECT_DIR}
          --log install_times.json
@@ -40,10 +41,10 @@ ci:
      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
      tags: ["aws"]
      script:
-      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
        - /sign.sh
-        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-        - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
+        - aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"
@@ -54,14 +55,14 @@ ci:
      before_script:
      - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
        - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
-        - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
+        - export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
      script:
      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
-        - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
-        - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
+        - echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
+        - spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
        - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
-        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
-        - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
+        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
+        - spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
      when: "always"
      retry:
        max: 2
@@ -89,6 +90,7 @@ ci:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

+  # TODO: Remove this block in Spack 0.23
  - cleanup-job:
      tags: ["service"]
      variables:
--- a/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
+++ b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
@@ -0,0 +1,11 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
--- a/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
+++ b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
@@ -0,0 +1,16 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-shared:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
--- a/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
--- a/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@@ -131,9 +131,6 @@ spack:
    - - $compiler
    - - $target

-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@@ -142,9 +142,6 @@ spack:
    - - $compiler
    - - $target

-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
@@ -30,8 +30,6 @@ spack:
  - $optimized_configs
  # - $optimized_libs

-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
@@ -30,9 +30,6 @@ spack:
  - $optimized_configs
  - $optimized_libs

-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -30,9 +30,6 @@ spack:
  - $optimized_configs
  - $optimized_libs

-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
@@ -30,8 +30,6 @@ spack:
  - $optimized_configs
  # - $optimized_libs

-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -21,7 +21,5 @@ spack:
        - - $default_specs
        - - $arch

-  mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
-
  cdash:
    build-group: Build Systems
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -58,8 +58,6 @@ spack:
    - ["~paraview +visit"]
    - [$^visit_specs]

-  mirrors: {mirror: s3://spack-binaries/develop/data-vis-sdk}
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -172,7 +172,5 @@ spack:
  # - variorum            # variorum: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/lib64/libpals.so.0: undefined reference to `json_array_append_new@@libjansson.so.4'
  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # openblas: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.; make[2]: *** [<builtin>: spotrf2.o] Error 1; make[1]: *** [Makefile:27: lapacklib] Error 2; make: *** [Makefile:250: netlib] Error 2

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-rhel" }
-
  cdash:
    build-group: E4S Cray
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -171,7 +171,5 @@ spack:
  # - variorum
  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-sles" }
-
  cdash:
    build-group: E4S Cray SLES
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@@ -340,8 +340,6 @@ spack:
  # - tasmanian +cuda cuda_arch=90  # tasmanian: conflicts with cuda@12
  # - upcxx +cuda cuda_arch=90      # upcxx: needs NVIDIA driver

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-arm-neoverse_v1" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -236,8 +236,6 @@ spack:

  - py-scipy

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -251,8 +251,6 @@ spack:
  # - trilinos +cuda cuda_arch=70           # trilinos: https://github.com/trilinos/Trilinos/issues/11630
  # - upcxx +cuda cuda_arch=70              # upcxx: needs NVIDIA driver

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@@ -335,8 +335,6 @@ spack:
  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-rocm-external" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -382,8 +382,6 @@ spack:
  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898

-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
@@ -49,8 +49,6 @@ spack:
  # FAILURES
  # - kokkos +wrapper +cuda cuda_arch=80 ^cuda@12.0.0     # https://github.com/spack/spack/issues/35378

-  mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -82,8 +82,6 @@ spack:
  # - r-xgboost
  - xgboost

-  mirrors: { "mirror": "s3://spack-binaries/develop/ml-darwin-aarch64-mps" }
-
  ci:
    pipeline-gen:
    - build-job-remove:
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@@ -76,9 +76,6 @@ spack:
    # - r-xgboost
    - xgboost

-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cpu
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@@ -79,9 +79,6 @@ spack:
    # - r-xgboost
    - xgboost

-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cuda
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
@@ -82,9 +82,6 @@ spack:
    # - r-xgboost
    - xgboost

-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-rocm
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
@@ -38,8 +38,6 @@ spack:
    - - $compiler
    - - $target

-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@@ -28,7 +28,7 @@ spack:
    - mfem +cuda ^hypre+cuda
    - raja
    - raja +cuda
-    - umpire 
+    - umpire
    - umpire +cuda

  - compiler:
@@ -44,8 +44,6 @@ spack:
    - - $compiler
    - - $target

-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" }
-
  ci:
    pipeline-gen:
    - build-job:
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -40,9 +40,6 @@ spack:
      - xbraid
      - zfp

-  mirrors:
-    mirror: "s3://spack-binaries/develop/radiuss"
-
  specs:
  - matrix:
    - [$radiuss]
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -49,8 +49,6 @@ spack:
  - $clang_packages
  - $gcc_spack_built_packages

-  mirrors:
-    mirror: s3://spack-binaries/develop/tutorial
  ci:
    pipeline-gen:
    - build-job: