gitlab ci: Rework how mirrors are configured (#39939)
Improve how mirrors are used in gitlab ci, where we have until now thought
of them as only a string.
By configuring ci mirrors ahead of time using the proposed mirror templates,
and by taking advantage of the expressiveness that spack now has for mirrors,
this PR will allow us to easily switch the protocol/url we use for fetching
binary dependencies.
This change also deprecates some gitlab functionality and marks it for
removal in Spack 0.23:
- arguments to "spack ci generate":
* --buildcache-destination
* --copy-to
- gitlab configuration options:
* enable-artifacts-buildcache
* temporary-storage-url-prefix
This commit is contained in:
Scott Wittenburg
GitHub
parent
b2d3e01fe6
commit
46c1a8e4c6
@ -213,6 +213,16 @@ pipeline jobs.
|
||||
``spack ci generate``
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Throughout this documentation, references to the "mirror" mean the target
|
||||
mirror which is checked for the presence of up-to-date specs, and where
|
||||
any scheduled jobs should push built binary packages. In the past, this
|
||||
defaulted to the mirror at index 0 in the mirror configs, and could be
|
||||
overridden using the ``--buildcache-destination`` argument. Starting with
|
||||
Spack 0.23, ``spack ci generate`` will require you to identify this mirror
|
||||
by the name "buildcache-destination". While you can configure any number
|
||||
of mirrors as sources for your pipelines, you will need to identify the
|
||||
destination mirror by name.
|
||||
|
||||
Concretizes the specs in the active environment, stages them (as described in
|
||||
:ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk.
|
||||
During concretization of the environment, ``spack ci generate`` also writes a
|
||||
|
||||
@ -49,6 +49,7 @@
|
||||
|
||||
TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror"
|
||||
SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
|
||||
# TODO: Remove this in Spack 0.23
|
||||
SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
|
||||
JOB_NAME_FORMAT = (
|
||||
"{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}"
|
||||
@ -678,7 +679,7 @@ def generate_gitlab_ci_yaml(
|
||||
remote_mirror_override (str): Typically only needed when one spack.yaml
|
||||
is used to populate several mirrors with binaries, based on some
|
||||
criteria. Spack protected pipelines populate different mirrors based
|
||||
on branch name, facilitated by this option.
|
||||
on branch name, facilitated by this option. DEPRECATED
|
||||
"""
|
||||
with spack.concretize.disable_compiler_existence_check():
|
||||
with env.write_transaction():
|
||||
@ -775,17 +776,39 @@ def generate_gitlab_ci_yaml(
|
||||
"instead.",
|
||||
)
|
||||
|
||||
if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1:
|
||||
tty.die("spack ci generate requires an env containing a mirror")
|
||||
pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
|
||||
deprecated_mirror_config = False
|
||||
buildcache_destination = None
|
||||
if "buildcache-destination" in pipeline_mirrors:
|
||||
if remote_mirror_override:
|
||||
tty.die(
|
||||
"Using the deprecated --buildcache-destination cli option and "
|
||||
"having a mirror named 'buildcache-destination' at the same time "
|
||||
"is not allowed"
|
||||
)
|
||||
buildcache_destination = pipeline_mirrors["buildcache-destination"]
|
||||
else:
|
||||
deprecated_mirror_config = True
|
||||
# TODO: This will be an error in Spack 0.23
|
||||
|
||||
ci_mirrors = yaml_root["mirrors"]
|
||||
mirror_urls = [url for url in ci_mirrors.values()]
|
||||
remote_mirror_url = mirror_urls[0]
|
||||
# TODO: Remove this block in spack 0.23
|
||||
remote_mirror_url = None
|
||||
if deprecated_mirror_config:
|
||||
if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1:
|
||||
tty.die("spack ci generate requires an env containing a mirror")
|
||||
|
||||
ci_mirrors = yaml_root["mirrors"]
|
||||
mirror_urls = [url for url in ci_mirrors.values()]
|
||||
remote_mirror_url = mirror_urls[0]
|
||||
|
||||
spack_buildcache_copy = os.environ.get("SPACK_COPY_BUILDCACHE", None)
|
||||
if spack_buildcache_copy:
|
||||
buildcache_copies = {}
|
||||
buildcache_copy_src_prefix = remote_mirror_override or remote_mirror_url
|
||||
buildcache_copy_src_prefix = (
|
||||
buildcache_destination.fetch_url
|
||||
if buildcache_destination
|
||||
else remote_mirror_override or remote_mirror_url
|
||||
)
|
||||
buildcache_copy_dest_prefix = spack_buildcache_copy
|
||||
|
||||
# Check for a list of "known broken" specs that we should not bother
|
||||
@ -797,6 +820,7 @@ def generate_gitlab_ci_yaml(
|
||||
|
||||
enable_artifacts_buildcache = False
|
||||
if "enable-artifacts-buildcache" in ci_config:
|
||||
tty.warn("Support for enable-artifacts-buildcache will be removed in Spack 0.23")
|
||||
enable_artifacts_buildcache = ci_config["enable-artifacts-buildcache"]
|
||||
|
||||
rebuild_index_enabled = True
|
||||
@ -805,13 +829,15 @@ def generate_gitlab_ci_yaml(
|
||||
|
||||
temp_storage_url_prefix = None
|
||||
if "temporary-storage-url-prefix" in ci_config:
|
||||
tty.warn("Support for temporary-storage-url-prefix will be removed in Spack 0.23")
|
||||
temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"]
|
||||
|
||||
# If a remote mirror override (alternate buildcache destination) was
|
||||
# specified, add it here in case it has already built hashes we might
|
||||
# generate.
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
mirrors_to_check = None
|
||||
if remote_mirror_override:
|
||||
if deprecated_mirror_config and remote_mirror_override:
|
||||
if spack_pipeline_type == "spack_protected_branch":
|
||||
# Overriding the main mirror in this case might result
|
||||
# in skipping jobs on a release pipeline because specs are
|
||||
@ -831,8 +857,9 @@ def generate_gitlab_ci_yaml(
|
||||
cfg.default_modify_scope(),
|
||||
)
|
||||
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
shared_pr_mirror = None
|
||||
if spack_pipeline_type == "spack_pull_request":
|
||||
if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request":
|
||||
stack_name = os.environ.get("SPACK_CI_STACK_NAME", "")
|
||||
shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name)
|
||||
spack.mirror.add(
|
||||
@ -884,6 +911,7 @@ def generate_gitlab_ci_yaml(
|
||||
job_log_dir = os.path.join(pipeline_artifacts_dir, "logs")
|
||||
job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction")
|
||||
job_test_dir = os.path.join(pipeline_artifacts_dir, "tests")
|
||||
# TODO: Remove this line in Spack 0.23
|
||||
local_mirror_dir = os.path.join(pipeline_artifacts_dir, "mirror")
|
||||
user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data")
|
||||
|
||||
@ -898,11 +926,11 @@ def generate_gitlab_ci_yaml(
|
||||
rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir)
|
||||
rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir)
|
||||
rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir)
|
||||
# TODO: Remove this line in Spack 0.23
|
||||
rel_local_mirror_dir = os.path.join(local_mirror_dir, ci_project_dir)
|
||||
rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir)
|
||||
|
||||
# Speed up staging by first fetching binary indices from all mirrors
|
||||
# (including the override mirror we may have just added above).
|
||||
try:
|
||||
bindist.binary_index.update()
|
||||
except bindist.FetchCacheError as e:
|
||||
@ -1113,6 +1141,7 @@ def main_script_replacements(cmd):
|
||||
},
|
||||
)
|
||||
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if enable_artifacts_buildcache:
|
||||
bc_root = os.path.join(local_mirror_dir, "build_cache")
|
||||
job_object["artifacts"]["paths"].extend(
|
||||
@ -1142,10 +1171,12 @@ def main_script_replacements(cmd):
|
||||
_print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisions)
|
||||
|
||||
# Clean up remote mirror override if enabled
|
||||
if remote_mirror_override:
|
||||
spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
|
||||
if spack_pipeline_type == "spack_pull_request":
|
||||
spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if deprecated_mirror_config:
|
||||
if remote_mirror_override:
|
||||
spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
|
||||
if spack_pipeline_type == "spack_pull_request":
|
||||
spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
|
||||
|
||||
tty.debug("{0} build jobs generated in {1} stages".format(job_id, stage_id))
|
||||
|
||||
@ -1176,10 +1207,28 @@ def main_script_replacements(cmd):
|
||||
sync_job["needs"] = [
|
||||
{"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)}
|
||||
]
|
||||
|
||||
if "variables" not in sync_job:
|
||||
sync_job["variables"] = {}
|
||||
|
||||
sync_job["variables"]["SPACK_COPY_ONLY_DESTINATION"] = (
|
||||
buildcache_destination.fetch_url
|
||||
if buildcache_destination
|
||||
else remote_mirror_override or remote_mirror_url
|
||||
)
|
||||
|
||||
if "buildcache-source" in pipeline_mirrors:
|
||||
buildcache_source = pipeline_mirrors["buildcache-source"].fetch_url
|
||||
else:
|
||||
# TODO: Remove this condition in Spack 0.23
|
||||
buildcache_source = os.environ.get("SPACK_SOURCE_MIRROR", None)
|
||||
sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source
|
||||
|
||||
output_object["copy"] = sync_job
|
||||
job_id += 1
|
||||
|
||||
if job_id > 0:
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if temp_storage_url_prefix:
|
||||
# There were some rebuild jobs scheduled, so we will need to
|
||||
# schedule a job to clean up the temporary storage location
|
||||
@ -1213,6 +1262,13 @@ def main_script_replacements(cmd):
|
||||
signing_job["when"] = "always"
|
||||
signing_job["retry"] = {"max": 2, "when": ["always"]}
|
||||
signing_job["interruptible"] = True
|
||||
if "variables" not in signing_job:
|
||||
signing_job["variables"] = {}
|
||||
signing_job["variables"]["SPACK_BUILDCACHE_DESTINATION"] = (
|
||||
buildcache_destination.push_url # need the s3 url for aws s3 sync
|
||||
if buildcache_destination
|
||||
else remote_mirror_override or remote_mirror_url
|
||||
)
|
||||
|
||||
output_object["sign-pkgs"] = signing_job
|
||||
|
||||
@ -1221,13 +1277,13 @@ def main_script_replacements(cmd):
|
||||
stage_names.append("stage-rebuild-index")
|
||||
final_job = spack_ci_ir["jobs"]["reindex"]["attributes"]
|
||||
|
||||
index_target_mirror = mirror_urls[0]
|
||||
if remote_mirror_override:
|
||||
index_target_mirror = remote_mirror_override
|
||||
final_job["stage"] = "stage-rebuild-index"
|
||||
target_mirror = remote_mirror_override or remote_mirror_url
|
||||
if buildcache_destination:
|
||||
target_mirror = buildcache_destination.push_url
|
||||
final_job["script"] = _unpack_script(
|
||||
final_job["script"],
|
||||
op=lambda cmd: cmd.replace("{index_target_mirror}", index_target_mirror),
|
||||
op=lambda cmd: cmd.replace("{index_target_mirror}", target_mirror),
|
||||
)
|
||||
|
||||
final_job["when"] = "always"
|
||||
@ -1249,20 +1305,24 @@ def main_script_replacements(cmd):
|
||||
"SPACK_CONCRETE_ENV_DIR": rel_concrete_env_dir,
|
||||
"SPACK_VERSION": spack_version,
|
||||
"SPACK_CHECKOUT_VERSION": version_to_clone,
|
||||
# TODO: Remove this line in Spack 0.23
|
||||
"SPACK_REMOTE_MIRROR_URL": remote_mirror_url,
|
||||
"SPACK_JOB_LOG_DIR": rel_job_log_dir,
|
||||
"SPACK_JOB_REPRO_DIR": rel_job_repro_dir,
|
||||
"SPACK_JOB_TEST_DIR": rel_job_test_dir,
|
||||
# TODO: Remove this line in Spack 0.23
|
||||
"SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir,
|
||||
"SPACK_PIPELINE_TYPE": str(spack_pipeline_type),
|
||||
"SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"),
|
||||
# TODO: Remove this line in Spack 0.23
|
||||
"SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None",
|
||||
"SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag),
|
||||
"SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
|
||||
"SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"),
|
||||
}
|
||||
|
||||
if remote_mirror_override:
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if deprecated_mirror_config and remote_mirror_override:
|
||||
(output_object["variables"]["SPACK_REMOTE_MIRROR_OVERRIDE"]) = remote_mirror_override
|
||||
|
||||
spack_stack_name = os.environ.get("SPACK_CI_STACK_NAME", None)
|
||||
@ -2002,43 +2062,23 @@ def process_command(name, commands, repro_dir, run=True, exit_on_failure=True):
|
||||
|
||||
|
||||
def create_buildcache(
|
||||
input_spec: spack.spec.Spec,
|
||||
*,
|
||||
pipeline_mirror_url: Optional[str] = None,
|
||||
buildcache_mirror_url: Optional[str] = None,
|
||||
sign_binaries: bool = False,
|
||||
input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False
|
||||
) -> List[PushResult]:
|
||||
"""Create the buildcache at the provided mirror(s).
|
||||
|
||||
Arguments:
|
||||
input_spec: Installed spec to package and push
|
||||
buildcache_mirror_url: URL for the buildcache mirror
|
||||
pipeline_mirror_url: URL for the pipeline mirror
|
||||
destination_mirror_urls: List of urls to push to
|
||||
sign_binaries: Whether or not to sign buildcache entry
|
||||
|
||||
Returns: A list of PushResults, indicating success or failure.
|
||||
"""
|
||||
results = []
|
||||
|
||||
# Create buildcache in either the main remote mirror, or in the
|
||||
# per-PR mirror, if this is a PR pipeline
|
||||
if buildcache_mirror_url:
|
||||
for mirror_url in destination_mirror_urls:
|
||||
results.append(
|
||||
PushResult(
|
||||
success=push_mirror_contents(input_spec, buildcache_mirror_url, sign_binaries),
|
||||
url=buildcache_mirror_url,
|
||||
)
|
||||
)
|
||||
|
||||
# Create another copy of that buildcache in the per-pipeline
|
||||
# temporary storage mirror (this is only done if either
|
||||
# artifacts buildcache is enabled or a temporary storage url
|
||||
# prefix is set)
|
||||
if pipeline_mirror_url:
|
||||
results.append(
|
||||
PushResult(
|
||||
success=push_mirror_contents(input_spec, pipeline_mirror_url, sign_binaries),
|
||||
url=pipeline_mirror_url,
|
||||
success=push_mirror_contents(input_spec, mirror_url, sign_binaries), url=mirror_url
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@ -191,6 +191,14 @@ def ci_generate(args):
|
||||
"""
|
||||
env = spack.cmd.require_active_env(cmd_name="ci generate")
|
||||
|
||||
if args.copy_to:
|
||||
tty.warn("The flag --copy-to is deprecated and will be removed in Spack 0.23")
|
||||
|
||||
if args.buildcache_destination:
|
||||
tty.warn(
|
||||
"The flag --buildcache-destination is deprecated and will be removed in Spack 0.23"
|
||||
)
|
||||
|
||||
output_file = args.output_file
|
||||
copy_yaml_to = args.copy_to
|
||||
run_optimizer = args.optimize
|
||||
@ -264,12 +272,6 @@ def ci_rebuild(args):
|
||||
if not ci_config:
|
||||
tty.die("spack ci rebuild requires an env containing ci cfg")
|
||||
|
||||
tty.msg(
|
||||
"SPACK_BUILDCACHE_DESTINATION={0}".format(
|
||||
os.environ.get("SPACK_BUILDCACHE_DESTINATION", None)
|
||||
)
|
||||
)
|
||||
|
||||
# Grab the environment variables we need. These either come from the
|
||||
# pipeline generation step ("spack ci generate"), where they were written
|
||||
# out as variables, or else provided by GitLab itself.
|
||||
@ -277,6 +279,7 @@ def ci_rebuild(args):
|
||||
job_log_dir = os.environ.get("SPACK_JOB_LOG_DIR")
|
||||
job_test_dir = os.environ.get("SPACK_JOB_TEST_DIR")
|
||||
repro_dir = os.environ.get("SPACK_JOB_REPRO_DIR")
|
||||
# TODO: Remove this in Spack 0.23
|
||||
local_mirror_dir = os.environ.get("SPACK_LOCAL_MIRROR_DIR")
|
||||
concrete_env_dir = os.environ.get("SPACK_CONCRETE_ENV_DIR")
|
||||
ci_pipeline_id = os.environ.get("CI_PIPELINE_ID")
|
||||
@ -285,9 +288,12 @@ def ci_rebuild(args):
|
||||
job_spec_pkg_name = os.environ.get("SPACK_JOB_SPEC_PKG_NAME")
|
||||
job_spec_dag_hash = os.environ.get("SPACK_JOB_SPEC_DAG_HASH")
|
||||
spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE")
|
||||
# TODO: Remove this in Spack 0.23
|
||||
remote_mirror_override = os.environ.get("SPACK_REMOTE_MIRROR_OVERRIDE")
|
||||
# TODO: Remove this in Spack 0.23
|
||||
remote_mirror_url = os.environ.get("SPACK_REMOTE_MIRROR_URL")
|
||||
spack_ci_stack_name = os.environ.get("SPACK_CI_STACK_NAME")
|
||||
# TODO: Remove this in Spack 0.23
|
||||
shared_pr_mirror_url = os.environ.get("SPACK_CI_SHARED_PR_MIRROR_URL")
|
||||
rebuild_everything = os.environ.get("SPACK_REBUILD_EVERYTHING")
|
||||
require_signing = os.environ.get("SPACK_REQUIRE_SIGNING")
|
||||
@ -344,21 +350,36 @@ def ci_rebuild(args):
|
||||
|
||||
full_rebuild = True if rebuild_everything and rebuild_everything.lower() == "true" else False
|
||||
|
||||
pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
|
||||
deprecated_mirror_config = False
|
||||
buildcache_destination = None
|
||||
if "buildcache-destination" in pipeline_mirrors:
|
||||
buildcache_destination = pipeline_mirrors["buildcache-destination"]
|
||||
else:
|
||||
deprecated_mirror_config = True
|
||||
# TODO: This will be an error in Spack 0.23
|
||||
|
||||
# If no override url exists, then just push binary package to the
|
||||
# normal remote mirror url.
|
||||
# TODO: Remove in Spack 0.23
|
||||
buildcache_mirror_url = remote_mirror_override or remote_mirror_url
|
||||
if buildcache_destination:
|
||||
buildcache_mirror_url = buildcache_destination.push_url
|
||||
|
||||
# Figure out what is our temporary storage mirror: Is it artifacts
|
||||
# buildcache? Or temporary-storage-url-prefix? In some cases we need to
|
||||
# force something or pipelines might not have a way to propagate build
|
||||
# artifacts from upstream to downstream jobs.
|
||||
# TODO: Remove this in Spack 0.23
|
||||
pipeline_mirror_url = None
|
||||
|
||||
# TODO: Remove this in Spack 0.23
|
||||
temp_storage_url_prefix = None
|
||||
if "temporary-storage-url-prefix" in ci_config:
|
||||
temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"]
|
||||
pipeline_mirror_url = url_util.join(temp_storage_url_prefix, ci_pipeline_id)
|
||||
|
||||
# TODO: Remove this in Spack 0.23
|
||||
enable_artifacts_mirror = False
|
||||
if "enable-artifacts-buildcache" in ci_config:
|
||||
enable_artifacts_mirror = ci_config["enable-artifacts-buildcache"]
|
||||
@ -454,12 +475,14 @@ def ci_rebuild(args):
|
||||
# If we decided there should be a temporary storage mechanism, add that
|
||||
# mirror now so it's used when we check for a hash match already
|
||||
# built for this spec.
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if pipeline_mirror_url:
|
||||
mirror = spack.mirror.Mirror(pipeline_mirror_url, name=spack_ci.TEMP_STORAGE_MIRROR_NAME)
|
||||
spack.mirror.add(mirror, cfg.default_modify_scope())
|
||||
pipeline_mirrors.append(pipeline_mirror_url)
|
||||
|
||||
# Check configured mirrors for a built spec with a matching hash
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
mirrors_to_check = None
|
||||
if remote_mirror_override:
|
||||
if spack_pipeline_type == "spack_protected_branch":
|
||||
@ -477,7 +500,8 @@ def ci_rebuild(args):
|
||||
)
|
||||
pipeline_mirrors.append(remote_mirror_override)
|
||||
|
||||
if spack_pipeline_type == "spack_pull_request":
|
||||
# TODO: Remove this in Spack 0.23
|
||||
if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request":
|
||||
if shared_pr_mirror_url != "None":
|
||||
pipeline_mirrors.append(shared_pr_mirror_url)
|
||||
|
||||
@ -499,6 +523,7 @@ def ci_rebuild(args):
|
||||
tty.msg("No need to rebuild {0}, found hash match at: ".format(job_spec_pkg_name))
|
||||
for match in matches:
|
||||
tty.msg(" {0}".format(match["mirror_url"]))
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if enable_artifacts_mirror:
|
||||
matching_mirror = matches[0]["mirror_url"]
|
||||
build_cache_dir = os.path.join(local_mirror_dir, "build_cache")
|
||||
@ -513,7 +538,8 @@ def ci_rebuild(args):
|
||||
# only want to keep the mirror being used by the current pipeline as it's binary
|
||||
# package destination. This ensures that the when we rebuild everything, we only
|
||||
# consume binary dependencies built in this pipeline.
|
||||
if full_rebuild:
|
||||
# TODO: Remove this in Spack 0.23
|
||||
if deprecated_mirror_config and full_rebuild:
|
||||
spack_ci.remove_other_mirrors(pipeline_mirrors, cfg.default_modify_scope())
|
||||
|
||||
# No hash match anywhere means we need to rebuild spec
|
||||
@ -678,21 +704,25 @@ def ci_rebuild(args):
|
||||
# print out some instructions on how to reproduce this build failure
|
||||
# outside of the pipeline environment.
|
||||
if install_exit_code == 0:
|
||||
if buildcache_mirror_url or pipeline_mirror_url:
|
||||
for result in spack_ci.create_buildcache(
|
||||
input_spec=job_spec,
|
||||
buildcache_mirror_url=buildcache_mirror_url,
|
||||
pipeline_mirror_url=pipeline_mirror_url,
|
||||
sign_binaries=spack_ci.can_sign_binaries(),
|
||||
):
|
||||
msg = tty.msg if result.success else tty.warn
|
||||
msg(
|
||||
"{} {} to {}".format(
|
||||
"Pushed" if result.success else "Failed to push",
|
||||
job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()),
|
||||
result.url,
|
||||
)
|
||||
mirror_urls = [buildcache_mirror_url]
|
||||
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
if pipeline_mirror_url:
|
||||
mirror_urls.append(pipeline_mirror_url)
|
||||
|
||||
for result in spack_ci.create_buildcache(
|
||||
input_spec=job_spec,
|
||||
destination_mirror_urls=mirror_urls,
|
||||
sign_binaries=spack_ci.can_sign_binaries(),
|
||||
):
|
||||
msg = tty.msg if result.success else tty.warn
|
||||
msg(
|
||||
"{} {} to {}".format(
|
||||
"Pushed" if result.success else "Failed to push",
|
||||
job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()),
|
||||
result.url,
|
||||
)
|
||||
)
|
||||
|
||||
# If this is a develop pipeline, check if the spec that we just built is
|
||||
# on the broken-specs list. If so, remove it.
|
||||
|
||||
@ -141,6 +141,7 @@
|
||||
}
|
||||
)
|
||||
|
||||
# TODO: Remove in Spack 0.23
|
||||
ci_properties = {
|
||||
"anyOf": [
|
||||
{
|
||||
@ -166,6 +167,7 @@
|
||||
properties = {
|
||||
"ci": {
|
||||
"oneOf": [
|
||||
# TODO: Replace with core-shared-properties in Spack 0.23
|
||||
ci_properties,
|
||||
# Allow legacy format under `ci` for `config update ci`
|
||||
spack.schema.gitlab_ci.gitlab_ci_properties,
|
||||
|
||||
@ -451,9 +451,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey
|
||||
monkeypatch.setattr(spack.ci, "push_mirror_contents", lambda a, b, c: True)
|
||||
|
||||
results = ci.create_buildcache(
|
||||
None,
|
||||
buildcache_mirror_url="file:///fake-url-one",
|
||||
pipeline_mirror_url="file:///fake-url-two",
|
||||
None, destination_mirror_urls=["file:///fake-url-one", "file:///fake-url-two"]
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
@ -463,7 +461,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey
|
||||
assert result2.success
|
||||
assert result2.url == "file:///fake-url-two"
|
||||
|
||||
results = ci.create_buildcache(None, buildcache_mirror_url="file:///fake-url-one")
|
||||
results = ci.create_buildcache(None, destination_mirror_urls=["file:///fake-url-one"])
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
|
||||
@ -2209,3 +2209,50 @@ def test_gitlab_config_scopes(
|
||||
assert all([t in rebuild_tags for t in ["spack", "service"]])
|
||||
expected_vars = ["CI_JOB_SIZE", "KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST"]
|
||||
assert all([v in rebuild_vars for v in expected_vars])
|
||||
|
||||
|
||||
def test_ci_generate_mirror_config(
|
||||
tmpdir,
|
||||
mutable_mock_env_path,
|
||||
install_mockery,
|
||||
mock_packages,
|
||||
monkeypatch,
|
||||
ci_base_environment,
|
||||
mock_binary_index,
|
||||
):
|
||||
"""Make sure the correct mirror gets used as the buildcache destination"""
|
||||
filename = str(tmpdir.join("spack.yaml"))
|
||||
with open(filename, "w") as f:
|
||||
f.write(
|
||||
"""\
|
||||
spack:
|
||||
specs:
|
||||
- archive-files
|
||||
mirrors:
|
||||
some-mirror: file:///this/is/a/source/mirror
|
||||
buildcache-destination: file:///push/binaries/here
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- submapping:
|
||||
- match:
|
||||
- archive-files
|
||||
build-job:
|
||||
tags:
|
||||
- donotcare
|
||||
image: donotcare
|
||||
"""
|
||||
)
|
||||
|
||||
with tmpdir.as_cwd():
|
||||
env_cmd("create", "test", "./spack.yaml")
|
||||
outputfile = str(tmpdir.join(".gitlab-ci.yml"))
|
||||
|
||||
with ev.read("test"):
|
||||
ci_cmd("generate", "--output-file", outputfile)
|
||||
with open(outputfile) as of:
|
||||
pipeline_doc = syaml.load(of.read())
|
||||
assert "rebuild-index" in pipeline_doc
|
||||
reindex_job = pipeline_doc["rebuild-index"]
|
||||
assert "script" in reindex_job
|
||||
reindex_step = reindex_job["script"][0]
|
||||
assert "file:///push/binaries/here" in reindex_step
|
||||
|
||||
@ -3,6 +3,12 @@ stages: [ "generate", "build", "publish" ]
|
||||
variables:
|
||||
SPACK_DISABLE_LOCAL_CONFIG: "1"
|
||||
SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
|
||||
# PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
|
||||
PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
|
||||
PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
|
||||
# PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
|
||||
PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
|
||||
PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"
|
||||
|
||||
default:
|
||||
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
|
||||
@ -68,7 +74,9 @@ default:
|
||||
########################################
|
||||
.base-job:
|
||||
variables:
|
||||
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
|
||||
PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
|
||||
# TODO: We can remove this when we drop the "deprecated" stack
|
||||
PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
|
||||
|
||||
rules:
|
||||
- if: $CI_COMMIT_REF_NAME == "develop"
|
||||
@ -76,7 +84,7 @@ default:
|
||||
when: always
|
||||
variables:
|
||||
SPACK_PIPELINE_TYPE: "spack_protected_branch"
|
||||
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_REQUIRE_SIGNING: "True"
|
||||
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
|
||||
@ -86,7 +94,7 @@ default:
|
||||
when: always
|
||||
variables:
|
||||
SPACK_PIPELINE_TYPE: "spack_protected_branch"
|
||||
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_PRUNE_UNTOUCHED: "False"
|
||||
SPACK_PRUNE_UP_TO_DATE: "False"
|
||||
SPACK_REQUIRE_SIGNING: "True"
|
||||
@ -98,8 +106,8 @@ default:
|
||||
when: always
|
||||
variables:
|
||||
SPACK_PIPELINE_TYPE: "spack_copy_only"
|
||||
SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
|
||||
PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
|
||||
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
|
||||
OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
|
||||
@ -108,9 +116,16 @@ default:
|
||||
when: always
|
||||
variables:
|
||||
SPACK_PIPELINE_TYPE: "spack_pull_request"
|
||||
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
|
||||
# TODO: We can remove this when we drop the "deprecated" stack
|
||||
PUSH_BUILDCACHE_DEPRECATED: "${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
|
||||
SPACK_PRUNE_UNTOUCHED: "True"
|
||||
SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
|
||||
# TODO: Change sync script to include target in branch name. Then we could
|
||||
# TODO: have multiple types of "PR" pipeline here. It would be better if we could
|
||||
# TODO: keep just this one and use a regex to capture the target branch, but so
|
||||
# TODO: far gitlab doesn't support that.
|
||||
PR_TARGET_REF_NAME: "develop"
|
||||
PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
|
||||
AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
|
||||
OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
|
||||
@ -126,13 +141,15 @@ default:
|
||||
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
|
||||
- spack env activate --without-view .
|
||||
- export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
|
||||
- spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
|
||||
< "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
|
||||
- spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
|
||||
- spack
|
||||
--config-scope "${SPACK_CI_CONFIG_ROOT}"
|
||||
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
|
||||
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
|
||||
${CI_STACK_CONFIG_SCOPES}
|
||||
ci generate --check-index-only
|
||||
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
|
||||
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
|
||||
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
|
||||
after_script:
|
||||
@ -182,7 +199,7 @@ default:
|
||||
- spack env activate --without-view .
|
||||
- spack
|
||||
ci generate --check-index-only
|
||||
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
|
||||
--buildcache-destination "${PUSH_BUILDCACHE_DEPRECATED}"
|
||||
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
|
||||
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
|
||||
after_script:
|
||||
@ -219,8 +236,7 @@ protected-publish:
|
||||
max: 2
|
||||
when: ["runner_system_failure", "stuck_or_timeout_failure"]
|
||||
variables:
|
||||
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
|
||||
SPACK_PIPELINE_TYPE: "spack_protected_branch"
|
||||
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
|
||||
@ -253,11 +269,6 @@ protected-publish:
|
||||
# you should inlclude your custom definitions at the end of the of the
|
||||
# extends list.
|
||||
#
|
||||
# Also note that if extending .base-job, the mirror url given in your
|
||||
# spack.yaml should take the form:
|
||||
#
|
||||
# s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
|
||||
#
|
||||
########################################
|
||||
# My Super Cool Pipeline
|
||||
########################################
|
||||
|
||||
@ -21,7 +21,8 @@ ci:
|
||||
- k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
|
||||
- k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
|
||||
script::
|
||||
- - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
|
||||
- - spack config blame mirrors
|
||||
- spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
|
||||
- - spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
|
||||
--prefix /home/software/spack:${CI_PROJECT_DIR}
|
||||
--log install_times.json
|
||||
@ -40,10 +41,10 @@ ci:
|
||||
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
|
||||
tags: ["aws"]
|
||||
script:
|
||||
- - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
|
||||
- - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
|
||||
- /sign.sh
|
||||
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
|
||||
- aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
|
||||
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
|
||||
- aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
|
||||
id_tokens:
|
||||
GITLAB_OIDC_TOKEN:
|
||||
aud: "${OIDC_TOKEN_AUDIENCE}"
|
||||
@ -54,14 +55,14 @@ ci:
|
||||
before_script:
|
||||
- - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
|
||||
- if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
|
||||
- export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
|
||||
- export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
|
||||
script:
|
||||
- - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
|
||||
- echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
|
||||
- spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
|
||||
- echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
|
||||
- spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
|
||||
- curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
|
||||
- aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
|
||||
- spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
|
||||
- aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
|
||||
- spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
|
||||
when: "always"
|
||||
retry:
|
||||
max: 2
|
||||
@ -89,6 +90,7 @@ ci:
|
||||
GITLAB_OIDC_TOKEN:
|
||||
aud: "${OIDC_TOKEN_AUDIENCE}"
|
||||
|
||||
# TODO: Remove this block in Spack 0.23
|
||||
- cleanup-job:
|
||||
tags: ["service"]
|
||||
variables:
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
mirrors:
|
||||
buildcache-source:
|
||||
fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
|
||||
push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
buildcache-destination:
|
||||
fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
@ -0,0 +1,16 @@
|
||||
mirrors:
|
||||
buildcache-source:
|
||||
fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
buildcache-destination:
|
||||
fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
buildcache-shared:
|
||||
fetch: ${PR_MIRROR_FETCH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
|
||||
push: ${PR_MIRROR_PUSH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
@ -0,0 +1,6 @@
|
||||
mirrors:
|
||||
buildcache-destination:
|
||||
fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
@ -0,0 +1,6 @@
|
||||
mirrors:
|
||||
buildcache-destination:
|
||||
fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
|
||||
source: False
|
||||
binary: True
|
||||
@ -131,9 +131,6 @@ spack:
|
||||
- - $compiler
|
||||
- - $target
|
||||
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -142,9 +142,6 @@ spack:
|
||||
- - $compiler
|
||||
- - $target
|
||||
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -30,8 +30,6 @@ spack:
|
||||
- $optimized_configs
|
||||
# - $optimized_libs
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -30,9 +30,6 @@ spack:
|
||||
- $optimized_configs
|
||||
- $optimized_libs
|
||||
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -30,9 +30,6 @@ spack:
|
||||
- $optimized_configs
|
||||
- $optimized_libs
|
||||
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -30,8 +30,6 @@ spack:
|
||||
- $optimized_configs
|
||||
# - $optimized_libs
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -21,7 +21,5 @@ spack:
|
||||
- - $default_specs
|
||||
- - $arch
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
|
||||
|
||||
cdash:
|
||||
build-group: Build Systems
|
||||
|
||||
@ -58,8 +58,6 @@ spack:
|
||||
- ["~paraview +visit"]
|
||||
- [$^visit_specs]
|
||||
|
||||
mirrors: {mirror: s3://spack-binaries/develop/data-vis-sdk}
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -172,7 +172,5 @@ spack:
|
||||
# - variorum # variorum: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/lib64/libpals.so.0: undefined reference to `json_array_append_new@@libjansson.so.4'
|
||||
# - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # openblas: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.; make[2]: *** [<builtin>: spotrf2.o] Error 1; make[1]: *** [Makefile:27: lapacklib] Error 2; make: *** [Makefile:250: netlib] Error 2
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-rhel" }
|
||||
|
||||
cdash:
|
||||
build-group: E4S Cray
|
||||
|
||||
@ -171,7 +171,5 @@ spack:
|
||||
# - variorum
|
||||
# - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-sles" }
|
||||
|
||||
cdash:
|
||||
build-group: E4S Cray SLES
|
||||
|
||||
@ -340,8 +340,6 @@ spack:
|
||||
# - tasmanian +cuda cuda_arch=90 # tasmanian: conflicts with cuda@12
|
||||
# - upcxx +cuda cuda_arch=90 # upcxx: needs NVIDIA driver
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-arm-neoverse_v1" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -236,8 +236,6 @@ spack:
|
||||
|
||||
- py-scipy
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -251,8 +251,6 @@ spack:
|
||||
# - trilinos +cuda cuda_arch=70 # trilinos: https://github.com/trilinos/Trilinos/issues/11630
|
||||
# - upcxx +cuda cuda_arch=70 # upcxx: needs NVIDIA driver
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -335,8 +335,6 @@ spack:
|
||||
# - lbann ~cuda +rocm amdgpu_target=gfx90a # aluminum: https://github.com/spack/spack/issues/38807
|
||||
# - papi +rocm amdgpu_target=gfx90a # papi: https://github.com/spack/spack/issues/27898
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-rocm-external" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -382,8 +382,6 @@ spack:
|
||||
# - lbann ~cuda +rocm amdgpu_target=gfx90a # aluminum: https://github.com/spack/spack/issues/38807
|
||||
# - papi +rocm amdgpu_target=gfx90a # papi: https://github.com/spack/spack/issues/27898
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -49,8 +49,6 @@ spack:
|
||||
# FAILURES
|
||||
# - kokkos +wrapper +cuda cuda_arch=80 ^cuda@12.0.0 # https://github.com/spack/spack/issues/35378
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -82,8 +82,6 @@ spack:
|
||||
# - r-xgboost
|
||||
- xgboost
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/ml-darwin-aarch64-mps" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job-remove:
|
||||
|
||||
@ -76,9 +76,6 @@ spack:
|
||||
# - r-xgboost
|
||||
- xgboost
|
||||
|
||||
mirrors:
|
||||
mirror: s3://spack-binaries/develop/ml-linux-x86_64-cpu
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -79,9 +79,6 @@ spack:
|
||||
# - r-xgboost
|
||||
- xgboost
|
||||
|
||||
mirrors:
|
||||
mirror: s3://spack-binaries/develop/ml-linux-x86_64-cuda
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -82,9 +82,6 @@ spack:
|
||||
# - r-xgboost
|
||||
- xgboost
|
||||
|
||||
mirrors:
|
||||
mirror: s3://spack-binaries/develop/ml-linux-x86_64-rocm
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -38,8 +38,6 @@ spack:
|
||||
- - $compiler
|
||||
- - $target
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -28,7 +28,7 @@ spack:
|
||||
- mfem +cuda ^hypre+cuda
|
||||
- raja
|
||||
- raja +cuda
|
||||
- umpire
|
||||
- umpire
|
||||
- umpire +cuda
|
||||
|
||||
- compiler:
|
||||
@ -44,8 +44,6 @@ spack:
|
||||
- - $compiler
|
||||
- - $target
|
||||
|
||||
mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" }
|
||||
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
@ -40,9 +40,6 @@ spack:
|
||||
- xbraid
|
||||
- zfp
|
||||
|
||||
mirrors:
|
||||
mirror: "s3://spack-binaries/develop/radiuss"
|
||||
|
||||
specs:
|
||||
- matrix:
|
||||
- [$radiuss]
|
||||
|
||||
@ -49,8 +49,6 @@ spack:
|
||||
- $clang_packages
|
||||
- $gcc_spack_built_packages
|
||||
|
||||
mirrors:
|
||||
mirror: s3://spack-binaries/develop/tutorial
|
||||
ci:
|
||||
pipeline-gen:
|
||||
- build-job:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user