zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Revert "gitlab: Add shared PR mirror to places pipelines look for binaries. (#33746)" (#34087)" (#34153)

Browse Source 
This reverts commit 63e4406514.
This commit is contained in:
Harmen Stoppels
2022-11-28 14:06:03 +01:00
committed by GitHub
parent 2a7ae2a700
commit efa1dba9e4
3 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/spack/spack/ci.py
View File

@@ -49,6 +49,7 @@
TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror" TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror"
SPACK_RESERVED_TAGS = ["public", "protected", "notary"] SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
spack_gpg = spack.main.SpackCommand("gpg") spack_gpg = spack.main.SpackCommand("gpg")
spack_compiler = spack.main.SpackCommand("compiler") spack_compiler = spack.main.SpackCommand("compiler")
@@ -729,6 +730,12 @@ def generate_gitlab_ci_yaml(
# won't fetch its index and include in our local cache. # won't fetch its index and include in our local cache.
spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope()) spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope())
shared_pr_mirror = None
if spack_pipeline_type == "spack_pull_request":
stack_name = os.environ.get("SPACK_CI_STACK_NAME", "")
shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name)
spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope())
pipeline_artifacts_dir = artifacts_root pipeline_artifacts_dir = artifacts_root
if not pipeline_artifacts_dir: if not pipeline_artifacts_dir:
proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd()) proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd())
@@ -803,6 +810,8 @@ def generate_gitlab_ci_yaml(
# Clean up remote mirror override if enabled # Clean up remote mirror override if enabled
if remote_mirror_override: if remote_mirror_override:
spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
if spack_pipeline_type == "spack_pull_request":
spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
all_job_names = [] all_job_names = []
output_object = {} output_object = {}
@@ -1292,6 +1301,7 @@ def generate_gitlab_ci_yaml(
"SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir,
"SPACK_PIPELINE_TYPE": str(spack_pipeline_type), "SPACK_PIPELINE_TYPE": str(spack_pipeline_type),
"SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"),
"SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None",
"SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag), "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag),
"SPACK_REBUILD_EVERYTHING": str(rebuild_everything), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
} }

5
lib/spack/spack/cmd/ci.py
View File

@@ -284,6 +284,7 @@ def ci_rebuild(args):
remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE") remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE")
remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL") remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL")
spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME") spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME")
shared_pr_mirror_url = get_env_var("SPACK_CI_SHARED_PR_MIRROR_URL")
rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING") rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING")
# Construct absolute paths relative to current $CI_PROJECT_DIR # Construct absolute paths relative to current $CI_PROJECT_DIR
@@ -471,6 +472,10 @@ def ci_rebuild(args):
spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope()) spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope())
pipeline_mirrors.append(remote_mirror_override) pipeline_mirrors.append(remote_mirror_override)
if spack_pipeline_type == "spack_pull_request":
if shared_pr_mirror_url != "None":
pipeline_mirrors.append(shared_pr_mirror_url)
matches = ( matches = (
None None
if full_rebuild if full_rebuild

2
lib/spack/spack/test/cmd/ci.py
View File

@@ -706,6 +706,8 @@ def test_ci_generate_for_pr_pipeline(
""" """
) )
monkeypatch.setattr(spack.ci, "SHARED_PR_MIRROR_URL", "https://fake.shared.pr.mirror")
with tmpdir.as_cwd(): with tmpdir.as_cwd():
env_cmd("create", "test", "./spack.yaml") env_cmd("create", "test", "./spack.yaml")
outputfile = str(tmpdir.join(".gitlab-ci.yml")) outputfile = str(tmpdir.join(".gitlab-ci.yml"))