This reverts commit 1462c35761.
This commit is contained in:
Harmen Stoppels
GitHub
parent
b7601f3042
commit
d45f682573
@ -37,7 +37,6 @@
|
||||
import spack.error
|
||||
import spack.main
|
||||
import spack.mirror
|
||||
import spack.package_base
|
||||
import spack.paths
|
||||
import spack.repo
|
||||
import spack.spec
|
||||
@ -265,22 +264,14 @@ def _format_job_needs(dep_jobs, build_group, prune_dag, rebuild_decisions):
|
||||
def get_change_revisions():
|
||||
"""If this is a git repo get the revisions to use when checking
|
||||
for changed packages and spack core modules."""
|
||||
rev1 = None
|
||||
rev2 = None
|
||||
|
||||
# Note: git_dir may be a file in a worktree. If it exists, attempt to use git
|
||||
# to determine if there is a revision
|
||||
git_dir = os.path.join(spack.paths.prefix, ".git")
|
||||
if os.path.exists(git_dir):
|
||||
# The default will only find changed packages from the last
|
||||
# commit. When the commit is a merge commit, this is will return all of the
|
||||
# changes on the topic.
|
||||
# TODO: Handle the case where the clone is not shallow clone of a merge commit
|
||||
# using `git merge-base`
|
||||
rev1 = "HEAD^"
|
||||
rev2 = "HEAD"
|
||||
|
||||
return rev1, rev2
|
||||
if os.path.exists(git_dir) and os.path.isdir(git_dir):
|
||||
# TODO: This will only find changed packages from the last
|
||||
# TODO: commit. While this may work for single merge commits
|
||||
# TODO: when merging the topic branch into the base, it will
|
||||
# TODO: require more thought outside of that narrow case.
|
||||
return "HEAD^", "HEAD"
|
||||
return None, None
|
||||
|
||||
|
||||
def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
|
||||
@ -399,7 +390,7 @@ class SpackCI:
|
||||
used by the CI generator(s).
|
||||
"""
|
||||
|
||||
def __init__(self, ci_config, spec_labels=None, stages=None):
|
||||
def __init__(self, ci_config, spec_labels, stages):
|
||||
"""Given the information from the ci section of the config
|
||||
and the staged jobs, set up meta data needed for generating Spack
|
||||
CI IR.
|
||||
@ -417,9 +408,8 @@ def __init__(self, ci_config, spec_labels=None, stages=None):
|
||||
}
|
||||
jobs = self.ir["jobs"]
|
||||
|
||||
if spec_labels and stages:
|
||||
for spec, dag_hash in _build_jobs(spec_labels, stages):
|
||||
jobs[dag_hash] = self.__init_job(spec)
|
||||
for spec, dag_hash in _build_jobs(spec_labels, stages):
|
||||
jobs[dag_hash] = self.__init_job(spec)
|
||||
|
||||
for name in self.named_jobs:
|
||||
# Skip the special named jobs
|
||||
@ -715,53 +705,14 @@ def generate_gitlab_ci_yaml(
|
||||
files (spack.yaml, spack.lock), etc should be written. GitLab
|
||||
requires this to be within the project directory.
|
||||
"""
|
||||
rev1, rev2 = get_change_revisions()
|
||||
tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}")
|
||||
|
||||
# Get the joined "ci" config with all of the current scopes resolved
|
||||
ci_config = cfg.get("ci")
|
||||
spack_prune_untouched = os.environ.get("SPACK_PRUNE_UNTOUCHED", None)
|
||||
|
||||
changed = rev1 and rev2
|
||||
affected_pkgs = None
|
||||
if spack_prune_untouched and changed:
|
||||
affected_pkgs = compute_affected_packages(rev1, rev2)
|
||||
tty.debug("affected pkgs:")
|
||||
if affected_pkgs:
|
||||
for p in affected_pkgs:
|
||||
tty.debug(f" {p}")
|
||||
else:
|
||||
tty.debug(" no affected packages...")
|
||||
|
||||
possible_builds = spack.package_base.possible_dependencies(*env.user_specs)
|
||||
changed = any((spec in p for p in possible_builds.values()) for spec in affected_pkgs)
|
||||
|
||||
if not changed:
|
||||
spack_ci = SpackCI(ci_config)
|
||||
spack_ci_ir = spack_ci.generate_ir()
|
||||
|
||||
# No jobs should be generated.
|
||||
noop_job = spack_ci_ir["jobs"]["noop"]["attributes"]
|
||||
# If this job fails ignore the status and carry on
|
||||
noop_job["retry"] = 0
|
||||
noop_job["allow_failure"] = True
|
||||
|
||||
tty.msg("Skipping concretization, generating no-op job")
|
||||
output_object = {"no-specs-to-rebuild": noop_job}
|
||||
|
||||
# Ensure the child pipeline always runs
|
||||
output_object["workflow"] = {"rules": [{"when": "always"}]}
|
||||
|
||||
with open(output_file, "w") as f:
|
||||
ruamel.yaml.YAML().dump(output_object, f)
|
||||
|
||||
return
|
||||
|
||||
with spack.concretize.disable_compiler_existence_check():
|
||||
with env.write_transaction():
|
||||
env.concretize()
|
||||
env.write()
|
||||
|
||||
# Get the joined "ci" config with all of the current scopes resolved
|
||||
ci_config = cfg.get("ci")
|
||||
|
||||
if not ci_config:
|
||||
raise SpackCIError("Environment does not have a `ci` configuration")
|
||||
|
||||
@ -786,13 +737,20 @@ def generate_gitlab_ci_yaml(
|
||||
dependent_depth = None
|
||||
|
||||
prune_untouched_packages = False
|
||||
spack_prune_untouched = os.environ.get("SPACK_PRUNE_UNTOUCHED", None)
|
||||
if spack_prune_untouched is not None and spack_prune_untouched.lower() == "true":
|
||||
# Requested to prune untouched packages, but assume we won't do that
|
||||
# unless we're actually in a git repo.
|
||||
if changed:
|
||||
rev1, rev2 = get_change_revisions()
|
||||
tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}")
|
||||
if rev1 and rev2:
|
||||
# If the stack file itself did not change, proceed with pruning
|
||||
if not get_stack_changed(env.manifest_path, rev1, rev2):
|
||||
prune_untouched_packages = True
|
||||
affected_pkgs = compute_affected_packages(rev1, rev2)
|
||||
tty.debug("affected pkgs:")
|
||||
for p in affected_pkgs:
|
||||
tty.debug(f" {p}")
|
||||
affected_specs = get_spec_filter_list(
|
||||
env, affected_pkgs, dependent_traverse_depth=dependent_depth
|
||||
)
|
||||
@ -1140,6 +1098,11 @@ def main_script_replacements(cmd):
|
||||
# warn only if there was actually a CDash configuration.
|
||||
tty.warn("Unable to populate buildgroup without CDash credentials")
|
||||
|
||||
service_job_retries = {
|
||||
"max": 2,
|
||||
"when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"],
|
||||
}
|
||||
|
||||
if copy_only_pipeline:
|
||||
stage_names.append("copy")
|
||||
sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"])
|
||||
@ -1199,10 +1162,7 @@ def main_script_replacements(cmd):
|
||||
)
|
||||
|
||||
final_job["when"] = "always"
|
||||
final_job["retry"] = {
|
||||
"max": 2,
|
||||
"when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"],
|
||||
}
|
||||
final_job["retry"] = service_job_retries
|
||||
final_job["interruptible"] = True
|
||||
final_job["dependencies"] = []
|
||||
|
||||
|
||||
@ -1650,45 +1650,3 @@ def fake_dyn_mapping_urlopener(*args, **kwargs):
|
||||
assert job.get("variables", {}).get("MY_VAR") == "hello"
|
||||
assert "ignored_field" not in job
|
||||
assert "unallowed_field" not in job
|
||||
|
||||
|
||||
def test_ci_generate_noop_no_concretize(
|
||||
tmpdir,
|
||||
working_env,
|
||||
mutable_mock_env_path,
|
||||
install_mockery,
|
||||
mock_packages,
|
||||
monkeypatch,
|
||||
ci_base_environment,
|
||||
):
|
||||
# Write the enviroment file
|
||||
filename = str(tmpdir.join("spack.yaml"))
|
||||
with open(filename, "w") as f:
|
||||
f.write(
|
||||
"""\
|
||||
spack:
|
||||
specs:
|
||||
- pkg-a
|
||||
mirrors:
|
||||
buildcache-destination: https://my.fake.mirror
|
||||
ci:
|
||||
type: gitlab
|
||||
"""
|
||||
)
|
||||
|
||||
def fake_compute_affected(r1=None, r2=None):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(ci, "compute_affected_packages", fake_compute_affected)
|
||||
monkeypatch.setenv("SPACK_PRUNE_UNTOUCHED", "TRUE") # enables pruning of untouched specs
|
||||
|
||||
with tmpdir.as_cwd():
|
||||
env_cmd("create", "test", "./spack.yaml")
|
||||
outputfile = str(tmpdir.join(".gitlab-ci.yml"))
|
||||
|
||||
with ev.read("test"):
|
||||
ci_cmd("generate", "--output-file", outputfile)
|
||||
|
||||
with open(outputfile) as of:
|
||||
pipeline_doc = syaml.load(of.read())
|
||||
assert "no-specs-to-rebuild" in pipeline_doc
|
||||
|
||||
Loading…
Reference in New Issue
Block a user