zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pipelines: Move PR testing stacks (currently only E4S) into spack (#21714)

Browse Source
This commit is contained in:
Scott Wittenburg
2021-02-18 18:50:57 -07:00
committed by GitHub
parent 0da2b82df2
commit 6b509a95da
6 changed files with 273 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
lib/spack/spack/ci.py
View File

@@ -22,7 +22,7 @@
import spack
import spack.binary_distribution as bindist
import spack.cmd.buildcache as buildcache
import spack.cmd
import spack.compilers as compilers
import spack.config as cfg
import spack.environment as ev
@@ -41,7 +41,7 @@
'always',
]
SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-pr-mirrors'
SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs'
TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror'
spack_gpg = spack.main.SpackCommand('gpg')
@@ -943,7 +943,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False,
cleanup_job['stage'] = 'cleanup-temp-storage'
cleanup_job['script'] = [
'spack mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
temp_storage_url_prefix)
]
cleanup_job['when'] = 'always'
@@ -1262,16 +1262,35 @@ def read_cdashid_from_mirror(spec, mirror_url):
def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id,
sign_binaries):
if mirror_url:
unsigned = not sign_binaries
tty.debug('Creating buildcache ({0})'.format(
'unsigned' if unsigned else 'signed'))
buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False,
output_location=mirror_url, force=True,
allow_root=True, unsigned=unsigned)
if build_id:
tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
build_id, mirror_url))
write_cdashid_to_mirror(build_id, spec, mirror_url)
try:
unsigned = not sign_binaries
tty.debug('Creating buildcache ({0})'.format(
'unsigned' if unsigned else 'signed'))
spack.cmd.buildcache._createtarball(
env, spec_yaml=yaml_path, add_deps=False,
output_location=mirror_url, force=True, allow_root=True,
unsigned=unsigned)
if build_id:
tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
build_id, mirror_url))
write_cdashid_to_mirror(build_id, spec, mirror_url)
except Exception as inst:
# If the mirror we're pushing to is on S3 and there's some
# permissions problem, for example, we can't just target
# that exception type here, since users of the
# `spack ci rebuild' may not need or want any dependency
# on boto3. So we use the first non-boto exception type
# in the heirarchy:
# boto3.exceptions.S3UploadFailedError
# boto3.exceptions.Boto3Error
# Exception
# BaseException
# object
err_msg = 'Error msg: {0}'.format(inst)
if 'Access Denied' in err_msg:
tty.msg('Permission problem writing to {0}'.format(
mirror_url))
tty.msg(err_msg)
def copy_stage_logs_to_artifacts(job_spec, job_log_dir):

31
lib/spack/spack/cmd/ci.py
View File

@@ -413,33 +413,18 @@ def add_mirror(mirror_name, mirror_url):
else:
buildcache_mirror_url = remote_mirror_url
try:
spack_ci.push_mirror_contents(
env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
cdash_build_id, sign_binaries)
except Exception as inst:
# If the mirror we're pushing to is on S3 and there's some
# permissions problem, for example, we can't just target
# that exception type here, since users of the
# `spack ci rebuild' may not need or want any dependency
# on boto3. So we use the first non-boto exception type
# in the heirarchy:
# boto3.exceptions.S3UploadFailedError
# boto3.exceptions.Boto3Error
# Exception
# BaseException
# object
err_msg = 'Error msg: {0}'.format(inst)
if 'Access Denied' in err_msg:
tty.msg('Permission problem writing to mirror')
tty.msg(err_msg)
# Create buildcache in either the main remote mirror, or in the
# per-PR mirror, if this is a PR pipeline
spack_ci.push_mirror_contents(
env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
cdash_build_id, sign_binaries)
# Create another copy of that buildcache in the per-pipeline
# temporary storage mirror (this is only done if either artifacts
# buildcache is enabled or a temporary storage url prefix is set)
spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path,
pipeline_mirror_url, cdash_build_id,
sign_binaries)
spack_ci.push_mirror_contents(
env, job_spec, job_spec_yaml_path, pipeline_mirror_url,
cdash_build_id, sign_binaries)
# Relate this build to its dependencies on CDash (if enabled)
if enable_cdash:

22
lib/spack/spack/test/cmd/ci.py
View File

@@ -878,6 +878,26 @@ def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
assert(len(dl_dir_list) == 3)
def test_push_mirror_contents_exceptions(monkeypatch, capsys):
def faked(env, spec_yaml=None, packages=None, add_spec=True,
add_deps=True, output_location=os.getcwd(),
signing_key=None, force=False, make_relative=False,
unsigned=False, allow_root=False, rebuild_index=False):
raise Exception('Error: Access Denied')
import spack.cmd.buildcache as buildcache
monkeypatch.setattr(buildcache, '_createtarball', faked)
url = 'fakejunk'
ci.push_mirror_contents(None, None, None, url, None, None)
captured = capsys.readouterr()
std_out = captured[0]
expect_msg = 'Permission problem writing to {0}'.format(url)
assert(expect_msg in std_out)
def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
@@ -1373,7 +1393,7 @@ def test_ci_generate_temp_storage_url(tmpdir, mutable_mock_env_path,
assert('script' in cleanup_job)
cleanup_task = cleanup_job['script'][0]
assert(cleanup_task.startswith('spack mirror destroy'))
assert(cleanup_task.startswith('spack -d mirror destroy'))
assert('stages' in pipeline_doc)
stages = pipeline_doc['stages']