zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pipelines: use shared pr mirror for pipeline generation and builds (#25529)

Browse Source 
Once PR binary graduation is deployed, the shared PR mirror will
contain binaries just built by a merged PR, before the subsequent
develop pipeline has had time to finish.  Using the shared PR mirror
as a source of binaries will reduce the number of times we have to
rebuild the same full hash.
This commit is contained in:
Scott Wittenburg 2021-08-20 07:45:23 -06:00 committed by GitHub
parent 37e4d32d53
commit 84100afc91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/spack/spack/ci.py
View File

@ -45,6 +45,8 @@
] ]
SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs' SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs'
SPACK_SHARED_PR_MIRROR_URL = url_util.join(SPACK_PR_MIRRORS_ROOT_URL,
'shared_pr_mirror')
TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror' TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror'
spack_gpg = spack.main.SpackCommand('gpg') spack_gpg = spack.main.SpackCommand('gpg')
@ -612,11 +614,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
'strip-compilers': False, 'strip-compilers': False,
}) })
# Add this mirror if it's enabled, as some specs might be up to date # Add per-PR mirror (and shared PR mirror) if enabled, as some specs might
# here and thus not need to be rebuilt. # be up to date in one of those and thus not need to be rebuilt.
if pr_mirror_url: if pr_mirror_url:
spack.mirror.add( spack.mirror.add(
'ci_pr_mirror', pr_mirror_url, cfg.default_modify_scope()) 'ci_pr_mirror', pr_mirror_url, cfg.default_modify_scope())
spack.mirror.add('ci_shared_pr_mirror',
SPACK_SHARED_PR_MIRROR_URL,
cfg.default_modify_scope())
pipeline_artifacts_dir = artifacts_root pipeline_artifacts_dir = artifacts_root
if not pipeline_artifacts_dir: if not pipeline_artifacts_dir:

12
lib/spack/spack/test/cmd/ci.py
View File

@ -617,6 +617,8 @@ def test_ci_generate_for_pr_pipeline(tmpdir, mutable_mock_env_path,
os.environ['SPACK_PR_BRANCH'] = 'fake-test-branch' os.environ['SPACK_PR_BRANCH'] = 'fake-test-branch'
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
try: try:
ci_cmd('generate', '--output-file', outputfile) ci_cmd('generate', '--output-file', outputfile)
finally: finally:
@ -668,6 +670,8 @@ def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path,
with ev.read('test'): with ev.read('test'):
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
ci_cmd('generate', '--output-file', outputfile) ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f: with open(outputfile) as f:
@ -1153,6 +1157,8 @@ def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path,
spack.main, 'get_version', lambda: '0.15.3-416-12ad69eb1') spack.main, 'get_version', lambda: '0.15.3-416-12ad69eb1')
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
ci_cmd('generate', '--output-file', outputfile) ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f: with open(outputfile) as f:
@ -1257,6 +1263,8 @@ def test_ci_generate_with_workarounds(tmpdir, mutable_mock_env_path,
with ev.read('test'): with ev.read('test'):
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
ci_cmd('generate', '--output-file', outputfile, '--dependencies') ci_cmd('generate', '--output-file', outputfile, '--dependencies')
with open(outputfile) as f: with open(outputfile) as f:
@ -1417,6 +1425,8 @@ def fake_get_mirrors_for_spec(spec=None, full_hash_match=False,
with ev.read('test'): with ev.read('test'):
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
ci_cmd('generate', '--output-file', outputfile) ci_cmd('generate', '--output-file', outputfile)
@ -1568,6 +1578,8 @@ def test_ci_generate_temp_storage_url(tmpdir, mutable_mock_env_path,
monkeypatch.setattr( monkeypatch.setattr(
ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror")
monkeypatch.setattr(
ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two")
with ev.read('test'): with ev.read('test'):
ci_cmd('generate', '--output-file', outputfile) ci_cmd('generate', '--output-file', outputfile)