From 5c4137baf19b9e271a2f13e886d6b875aab067dd Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 14 Nov 2022 19:37:23 -0700 Subject: [PATCH] gitlab: Add shared PR mirror to places pipelines look for binaries. (#33746) While binaries built for PRs that get merged must still be rebuilt in develop pipelines, they can be used by other PRs that find they would otherwise need to rebuild them. Now that spackbot is managing copying PR binaries from merged PRs into a shared location, keeping it pruned to a reasonable size, and making sure the indices are up to date, spack can use these mirrors as a potential source of binaries. --- lib/spack/spack/ci.py | 10 ++++++++++ lib/spack/spack/cmd/ci.py | 5 +++++ lib/spack/spack/test/cmd/ci.py | 2 ++ 3 files changed, 17 insertions(+) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index aa54b71b3d4..d6113c0dc94 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -51,6 +51,7 @@ TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror" SPACK_RESERVED_TAGS = ["public", "protected", "notary"] +SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror" spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") @@ -731,6 +732,12 @@ def generate_gitlab_ci_yaml( # won't fetch its index and include in our local cache. spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope()) + shared_pr_mirror = None + if spack_pipeline_type == "spack_pull_request": + stack_name = os.environ.get("SPACK_CI_STACK_NAME", "") + shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name) + spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope()) + pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd()) @@ -805,6 +812,8 @@ def generate_gitlab_ci_yaml( # Clean up remote mirror override if enabled if remote_mirror_override: spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) + if spack_pipeline_type == "spack_pull_request": + spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) all_job_names = [] output_object = {} @@ -1294,6 +1303,7 @@ def generate_gitlab_ci_yaml( "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_PIPELINE_TYPE": str(spack_pipeline_type), "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), + "SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None", "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything), } diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 6f29da1f4d6..168455bf9c5 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -285,6 +285,7 @@ def ci_rebuild(args): remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE") remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL") spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME") + shared_pr_mirror_url = get_env_var("SPACK_CI_SHARED_PR_MIRROR_URL") rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING") # Construct absolute paths relative to current $CI_PROJECT_DIR @@ -472,6 +473,10 @@ def ci_rebuild(args): spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope()) pipeline_mirrors.append(remote_mirror_override) + if spack_pipeline_type == "spack_pull_request": + if shared_pr_mirror_url != "None": + pipeline_mirrors.append(shared_pr_mirror_url) + matches = ( None if full_rebuild diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 73b474c7dfd..d01e1e082db 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -706,6 +706,8 @@ def test_ci_generate_for_pr_pipeline( """ ) + monkeypatch.setattr(spack.ci, "SHARED_PR_MIRROR_URL", "https://fake.shared.pr.mirror") + with tmpdir.as_cwd(): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmpdir.join(".gitlab-ci.yml"))