Extract depfile logic from cli command into a core module (#36995)
This commit is contained in:
Harmen Stoppels
GitHub
parent
3a5e48f476
commit
ae909b3688
@ -4,7 +4,6 @@
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
@ -24,10 +23,11 @@
|
||||
import spack.cmd.uninstall
|
||||
import spack.config
|
||||
import spack.environment as ev
|
||||
import spack.environment.depfile as depfile
|
||||
import spack.environment.shell
|
||||
import spack.schema.env
|
||||
import spack.spec
|
||||
import spack.tengine
|
||||
import spack.traverse as traverse
|
||||
import spack.util.string as string
|
||||
from spack.util.environment import EnvironmentModifications
|
||||
|
||||
@ -637,161 +637,23 @@ def env_depfile_setup_parser(subparser):
|
||||
)
|
||||
|
||||
|
||||
def _deptypes(use_buildcache):
|
||||
"""What edges should we follow for a given node? If it's a cache-only
|
||||
node, then we can drop build type deps."""
|
||||
return ("link", "run") if use_buildcache == "only" else ("build", "link", "run")
|
||||
|
||||
|
||||
class MakeTargetVisitor(object):
|
||||
"""This visitor produces an adjacency list of a (reduced) DAG, which
|
||||
is used to generate Makefile targets with their prerequisites."""
|
||||
|
||||
def __init__(self, target, pkg_buildcache, deps_buildcache):
|
||||
"""
|
||||
Args:
|
||||
target: function that maps dag_hash -> make target string
|
||||
pkg_buildcache (str): "only", "never", "auto": when "only",
|
||||
redundant build deps of roots are dropped
|
||||
deps_buildcache (str): same as pkg_buildcache, but for non-root specs.
|
||||
"""
|
||||
self.adjacency_list = []
|
||||
self.target = target
|
||||
self.pkg_buildcache = pkg_buildcache
|
||||
self.deps_buildcache = deps_buildcache
|
||||
self.deptypes_root = _deptypes(pkg_buildcache)
|
||||
self.deptypes_deps = _deptypes(deps_buildcache)
|
||||
|
||||
def neighbors(self, node):
|
||||
"""Produce a list of spec to follow from node"""
|
||||
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
|
||||
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
|
||||
|
||||
def build_cache_flag(self, depth):
|
||||
setting = self.pkg_buildcache if depth == 0 else self.deps_buildcache
|
||||
if setting == "only":
|
||||
return "--use-buildcache=only"
|
||||
elif setting == "never":
|
||||
return "--use-buildcache=never"
|
||||
return ""
|
||||
|
||||
def accept(self, node):
|
||||
fmt = "{name}-{version}-{hash}"
|
||||
tgt = node.edge.spec.format(fmt)
|
||||
spec_str = node.edge.spec.format(
|
||||
"{name}{@version}{%compiler}{variants}{arch=architecture}"
|
||||
)
|
||||
buildcache_flag = self.build_cache_flag(node.depth)
|
||||
prereqs = " ".join([self.target(dep.spec.format(fmt)) for dep in self.neighbors(node)])
|
||||
self.adjacency_list.append(
|
||||
(tgt, prereqs, node.edge.spec.dag_hash(), spec_str, buildcache_flag)
|
||||
)
|
||||
|
||||
# We already accepted this
|
||||
return True
|
||||
|
||||
|
||||
def env_depfile(args):
|
||||
# Currently only make is supported.
|
||||
spack.cmd.require_active_env(cmd_name="env depfile")
|
||||
env = ev.active_environment()
|
||||
|
||||
# Special make targets are useful when including a makefile in another, and you
|
||||
# need to "namespace" the targets to avoid conflicts.
|
||||
if args.make_prefix is None:
|
||||
prefix = os.path.join(env.env_subdir_path, "makedeps")
|
||||
else:
|
||||
prefix = args.make_prefix
|
||||
|
||||
def get_target(name):
|
||||
# The `all` and `clean` targets are phony. It doesn't make sense to
|
||||
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
|
||||
# sense to have a prefix like `env/all`, `env/clean` when they are
|
||||
# supposed to be included
|
||||
if name in ("all", "clean") and os.path.isabs(prefix):
|
||||
return name
|
||||
else:
|
||||
return os.path.join(prefix, name)
|
||||
|
||||
def get_install_target(name):
|
||||
return os.path.join(prefix, "install", name)
|
||||
|
||||
def get_install_deps_target(name):
|
||||
return os.path.join(prefix, "install-deps", name)
|
||||
|
||||
# What things do we build when running make? By default, we build the
|
||||
# root specs. If specific specs are provided as input, we build those.
|
||||
if args.specs:
|
||||
abstract_specs = spack.cmd.parse_specs(args.specs)
|
||||
roots = [env.matching_spec(s) for s in abstract_specs]
|
||||
else:
|
||||
roots = [s for _, s in env.concretized_specs()]
|
||||
|
||||
# We produce a sub-DAG from the DAG induced by roots, where we drop build
|
||||
# edges for those specs that are installed through a binary cache.
|
||||
pkg_buildcache, dep_buildcache = args.use_buildcache
|
||||
make_targets = MakeTargetVisitor(get_install_target, pkg_buildcache, dep_buildcache)
|
||||
traverse.traverse_breadth_first_with_visitor(
|
||||
roots, traverse.CoverNodesVisitor(make_targets, key=lambda s: s.dag_hash())
|
||||
)
|
||||
|
||||
# Root specs without deps are the prereqs for the environment target
|
||||
root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
|
||||
|
||||
all_pkg_identifiers = []
|
||||
|
||||
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
|
||||
# generated makefiles to add post-install hooks, like pushing to a buildcache,
|
||||
# running tests, etc.
|
||||
# NOTE: GNU Make allows directory separators in variable names, so for consistency
|
||||
# we can namespace this variable with the same prefix as targets.
|
||||
if args.make_prefix is None:
|
||||
pkg_identifier_variable = "SPACK_PACKAGE_IDS"
|
||||
else:
|
||||
pkg_identifier_variable = os.path.join(prefix, "SPACK_PACKAGE_IDS")
|
||||
|
||||
# All install and install-deps targets
|
||||
all_install_related_targets = []
|
||||
|
||||
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
|
||||
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
|
||||
# we don't have a custom make target prefix.
|
||||
phony_convenience_targets = []
|
||||
|
||||
for tgt, _, _, _, _ in make_targets.adjacency_list:
|
||||
all_pkg_identifiers.append(tgt)
|
||||
all_install_related_targets.append(get_install_target(tgt))
|
||||
all_install_related_targets.append(get_install_deps_target(tgt))
|
||||
if args.make_prefix is None:
|
||||
phony_convenience_targets.append(os.path.join("install", tgt))
|
||||
phony_convenience_targets.append(os.path.join("install-deps", tgt))
|
||||
|
||||
buf = io.StringIO()
|
||||
|
||||
filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None
|
||||
template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
|
||||
|
||||
rendered = template.render(
|
||||
{
|
||||
"all_target": get_target("all"),
|
||||
"env_target": get_target("env"),
|
||||
"clean_target": get_target("clean"),
|
||||
"all_install_related_targets": " ".join(all_install_related_targets),
|
||||
"root_install_targets": " ".join(root_install_targets),
|
||||
"dirs_target": get_target("dirs"),
|
||||
"environment": env.path,
|
||||
"install_target": get_target("install"),
|
||||
"install_deps_target": get_target("install-deps"),
|
||||
"any_hash_target": get_target("%"),
|
||||
"jobserver_support": "+" if args.jobserver else "",
|
||||
"adjacency_list": make_targets.adjacency_list,
|
||||
"phony_convenience_targets": " ".join(phony_convenience_targets),
|
||||
"pkg_ids_variable": pkg_identifier_variable,
|
||||
"pkg_ids": " ".join(all_pkg_identifiers),
|
||||
}
|
||||
model = depfile.MakefileModel.from_env(
|
||||
ev.active_environment(),
|
||||
filter_specs=filter_specs,
|
||||
pkg_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[0]),
|
||||
dep_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[1]),
|
||||
make_prefix=args.make_prefix,
|
||||
jobserver=args.jobserver,
|
||||
)
|
||||
|
||||
buf.write(rendered)
|
||||
makefile = buf.getvalue()
|
||||
makefile = template.render(model.to_dict())
|
||||
|
||||
# Finally write to stdout/file.
|
||||
if args.output:
|
||||
|
||||
239
lib/spack/spack/environment/depfile.py
Normal file
239
lib/spack/spack/environment/depfile.py
Normal file
@ -0,0 +1,239 @@
|
||||
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
"""
|
||||
This module contains the traversal logic and models that can be used to generate
|
||||
depfiles from an environment.
|
||||
"""
|
||||
|
||||
import os
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
|
||||
import spack.environment.environment as ev
|
||||
import spack.spec
|
||||
import spack.traverse as traverse
|
||||
|
||||
|
||||
class UseBuildCache(Enum):
|
||||
ONLY = 1
|
||||
NEVER = 2
|
||||
AUTO = 3
|
||||
|
||||
@staticmethod
|
||||
def from_string(s: str) -> "UseBuildCache":
|
||||
if s == "only":
|
||||
return UseBuildCache.ONLY
|
||||
elif s == "never":
|
||||
return UseBuildCache.NEVER
|
||||
elif s == "auto":
|
||||
return UseBuildCache.AUTO
|
||||
raise ValueError(f"invalid value for UseBuildCache: {s}")
|
||||
|
||||
|
||||
def _deptypes(use_buildcache: UseBuildCache):
|
||||
"""What edges should we follow for a given node? If it's a cache-only
|
||||
node, then we can drop build type deps."""
|
||||
return ("link", "run") if use_buildcache == UseBuildCache.ONLY else ("build", "link", "run")
|
||||
|
||||
|
||||
class DepfileNode:
|
||||
"""Contains a spec, a subset of its dependencies, and a flag whether it should be
|
||||
buildcache only/never/auto."""
|
||||
|
||||
def __init__(
|
||||
self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache
|
||||
):
|
||||
self.target = target
|
||||
self.prereqs = prereqs
|
||||
if buildcache == UseBuildCache.ONLY:
|
||||
self.buildcache_flag = "--use-buildcache=only"
|
||||
elif buildcache == UseBuildCache.NEVER:
|
||||
self.buildcache_flag = "--use-buildcache=never"
|
||||
else:
|
||||
self.buildcache_flag = ""
|
||||
|
||||
|
||||
class DepfileSpecVisitor:
|
||||
"""This visitor produces an adjacency list of a (reduced) DAG, which
|
||||
is used to generate depfile targets with their prerequisites. Currently
|
||||
it only drops build deps when using buildcache only mode.
|
||||
|
||||
Note that the DAG could be reduced even more by dropping build edges of specs
|
||||
installed at the moment the depfile is generated, but that would produce
|
||||
stateful depfiles that would not fail when the database is wiped later."""
|
||||
|
||||
def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache):
|
||||
self.adjacency_list: List[DepfileNode] = []
|
||||
self.pkg_buildcache = pkg_buildcache
|
||||
self.deps_buildcache = deps_buildcache
|
||||
self.deptypes_root = _deptypes(pkg_buildcache)
|
||||
self.deptypes_deps = _deptypes(deps_buildcache)
|
||||
|
||||
def neighbors(self, node):
|
||||
"""Produce a list of spec to follow from node"""
|
||||
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
|
||||
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
|
||||
|
||||
def accept(self, node):
|
||||
self.adjacency_list.append(
|
||||
DepfileNode(
|
||||
target=node.edge.spec,
|
||||
prereqs=[edge.spec for edge in self.neighbors(node)],
|
||||
buildcache=self.pkg_buildcache if node.depth == 0 else self.deps_buildcache,
|
||||
)
|
||||
)
|
||||
|
||||
# We already accepted this
|
||||
return True
|
||||
|
||||
|
||||
class MakefileModel:
|
||||
"""This class produces all data to render a makefile for specs of an environment."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
env: ev.Environment,
|
||||
roots: List[spack.spec.Spec],
|
||||
adjacency_list: List[DepfileNode],
|
||||
make_prefix: Optional[str],
|
||||
jobserver: bool,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
env: environment to generate the makefile for
|
||||
roots: specs that get built in the default target
|
||||
adjacency_list: list of DepfileNode, mapping specs to their dependencies
|
||||
make_prefix: prefix for makefile targets
|
||||
jobserver: when enabled, make will invoke Spack with jobserver support. For
|
||||
dry-run this should be disabled.
|
||||
"""
|
||||
# Currently we can only use depfile with an environment since Spack needs to
|
||||
# find the concrete specs somewhere.
|
||||
self.env_path = env.path
|
||||
|
||||
# These specs are built in the default target.
|
||||
self.roots = roots
|
||||
|
||||
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
|
||||
# generated makefiles to add post-install hooks, like pushing to a buildcache,
|
||||
# running tests, etc.
|
||||
if make_prefix is None:
|
||||
self.make_prefix = os.path.join(env.env_subdir_path, "makedeps")
|
||||
self.pkg_identifier_variable = "SPACK_PACKAGE_IDS"
|
||||
else:
|
||||
# NOTE: GNU Make allows directory separators in variable names, so for consistency
|
||||
# we can namespace this variable with the same prefix as targets.
|
||||
self.make_prefix = make_prefix
|
||||
self.pkg_identifier_variable = os.path.join(make_prefix, "SPACK_PACKAGE_IDS")
|
||||
|
||||
# And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag)
|
||||
self.make_adjacency_list = [
|
||||
(
|
||||
self._safe_name(item.target),
|
||||
" ".join(self._install_target(self._safe_name(s)) for s in item.prereqs),
|
||||
item.target.dag_hash(),
|
||||
item.target.format("{name}{@version}{%compiler}{variants}{arch=architecture}"),
|
||||
item.buildcache_flag,
|
||||
)
|
||||
for item in adjacency_list
|
||||
]
|
||||
|
||||
# Root specs without deps are the prereqs for the environment target
|
||||
self.root_install_targets = [self._install_target(self._safe_name(s)) for s in roots]
|
||||
|
||||
self.jobserver_support = "+" if jobserver else ""
|
||||
|
||||
# All package identifiers, used to generate the SPACK_PACKAGE_IDS variable
|
||||
self.all_pkg_identifiers: List[str] = []
|
||||
|
||||
# All install and install-deps targets
|
||||
self.all_install_related_targets: List[str] = []
|
||||
|
||||
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
|
||||
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
|
||||
# we don't have a custom make target prefix.
|
||||
self.phony_convenience_targets: List[str] = []
|
||||
|
||||
for node in adjacency_list:
|
||||
tgt = self._safe_name(node.target)
|
||||
self.all_pkg_identifiers.append(tgt)
|
||||
self.all_install_related_targets.append(self._install_target(tgt))
|
||||
self.all_install_related_targets.append(self._install_deps_target(tgt))
|
||||
if make_prefix is None:
|
||||
self.phony_convenience_targets.append(os.path.join("install", tgt))
|
||||
self.phony_convenience_targets.append(os.path.join("install-deps", tgt))
|
||||
|
||||
def _safe_name(self, spec: spack.spec.Spec) -> str:
|
||||
return spec.format("{name}-{version}-{hash}")
|
||||
|
||||
def _target(self, name: str) -> str:
|
||||
# The `all` and `clean` targets are phony. It doesn't make sense to
|
||||
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
|
||||
# sense to have a prefix like `env/all`, `env/clean` when they are
|
||||
# supposed to be included
|
||||
if name in ("all", "clean") and os.path.isabs(self.make_prefix):
|
||||
return name
|
||||
else:
|
||||
return os.path.join(self.make_prefix, name)
|
||||
|
||||
def _install_target(self, name: str) -> str:
|
||||
return os.path.join(self.make_prefix, "install", name)
|
||||
|
||||
def _install_deps_target(self, name: str) -> str:
|
||||
return os.path.join(self.make_prefix, "install-deps", name)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"all_target": self._target("all"),
|
||||
"env_target": self._target("env"),
|
||||
"clean_target": self._target("clean"),
|
||||
"all_install_related_targets": " ".join(self.all_install_related_targets),
|
||||
"root_install_targets": " ".join(self.root_install_targets),
|
||||
"dirs_target": self._target("dirs"),
|
||||
"environment": self.env_path,
|
||||
"install_target": self._target("install"),
|
||||
"install_deps_target": self._target("install-deps"),
|
||||
"any_hash_target": self._target("%"),
|
||||
"jobserver_support": self.jobserver_support,
|
||||
"adjacency_list": self.make_adjacency_list,
|
||||
"phony_convenience_targets": " ".join(self.phony_convenience_targets),
|
||||
"pkg_ids_variable": self.pkg_identifier_variable,
|
||||
"pkg_ids": " ".join(self.all_pkg_identifiers),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_env(
|
||||
env: ev.Environment,
|
||||
*,
|
||||
filter_specs: Optional[List[spack.spec.Spec]] = None,
|
||||
pkg_buildcache: UseBuildCache = UseBuildCache.AUTO,
|
||||
dep_buildcache: UseBuildCache = UseBuildCache.AUTO,
|
||||
make_prefix: Optional[str] = None,
|
||||
jobserver: bool = True,
|
||||
) -> "MakefileModel":
|
||||
"""Produces a MakefileModel from an environment and a list of specs.
|
||||
|
||||
Args:
|
||||
env: the environment to use
|
||||
filter_specs: if provided, only these specs will be built from the environment,
|
||||
otherwise the environment roots are used.
|
||||
pkg_buildcache: whether to only use the buildcache for top-level specs.
|
||||
dep_buildcache: whether to only use the buildcache for non-top-level specs.
|
||||
make_prefix: the prefix for the makefile targets
|
||||
jobserver: when enabled, make will invoke Spack with jobserver support. For
|
||||
dry-run this should be disabled.
|
||||
"""
|
||||
# If no specs are provided as a filter, build all the specs in the environment.
|
||||
if filter_specs:
|
||||
entrypoints = [env.matching_spec(s) for s in filter_specs]
|
||||
else:
|
||||
entrypoints = [s for _, s in env.concretized_specs()]
|
||||
|
||||
visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache)
|
||||
traverse.traverse_breadth_first_with_visitor(
|
||||
entrypoints, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash())
|
||||
)
|
||||
|
||||
return MakefileModel(env, entrypoints, visitor.adjacency_list, make_prefix, jobserver)
|
||||
@ -3025,6 +3025,15 @@ def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_m
|
||||
assert current_versions == expected_versions
|
||||
|
||||
|
||||
def _parse_dry_run_package_installs(make_output):
|
||||
"""Parse `spack install ... # <spec>` output from a make dry run."""
|
||||
return [
|
||||
Spec(line.split("# ")[1]).name
|
||||
for line in make_output.splitlines()
|
||||
if line.startswith("spack")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"depfile_flags,expected_installs",
|
||||
[
|
||||
@ -3108,15 +3117,66 @@ def test_environment_depfile_makefile(depfile_flags, expected_installs, tmpdir,
|
||||
# Do make dry run.
|
||||
out = make("-n", "-f", makefile, output=str)
|
||||
|
||||
# Spack install commands are of the form "spack install ... # <spec>",
|
||||
# so we just parse the spec again, for simplicity.
|
||||
specs_that_make_would_install = [
|
||||
Spec(line.split("# ")[1]).name for line in out.splitlines() if line.startswith("spack")
|
||||
]
|
||||
specs_that_make_would_install = _parse_dry_run_package_installs(out)
|
||||
|
||||
# Check that all specs are there (without duplicates)
|
||||
assert set(specs_that_make_would_install) == set(expected_installs)
|
||||
assert len(specs_that_make_would_install) == len(expected_installs)
|
||||
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"picked_package,expected_installs",
|
||||
[
|
||||
(
|
||||
"dttop",
|
||||
[
|
||||
"dtbuild2",
|
||||
"dtlink2",
|
||||
"dtrun2",
|
||||
"dtbuild1",
|
||||
"dtlink4",
|
||||
"dtlink3",
|
||||
"dtlink1",
|
||||
"dtlink5",
|
||||
"dtbuild3",
|
||||
"dtrun3",
|
||||
"dtrun1",
|
||||
"dttop",
|
||||
],
|
||||
),
|
||||
("dtrun1", ["dtlink5", "dtbuild3", "dtrun3", "dtrun1"]),
|
||||
],
|
||||
)
|
||||
def test_depfile_phony_convenience_targets(
|
||||
picked_package, expected_installs: set, tmpdir, mock_packages
|
||||
):
|
||||
"""Check whether convenience targets "install/%" and "install-deps/%" are created for
|
||||
each package if "--make-prefix" is absent."""
|
||||
make = Executable("make")
|
||||
with fs.working_dir(str(tmpdir)):
|
||||
with ev.Environment("."):
|
||||
add("dttop")
|
||||
concretize()
|
||||
|
||||
with ev.Environment(".") as e:
|
||||
picked_spec = e.matching_spec(picked_package)
|
||||
env("depfile", "-o", "Makefile", "--make-disable-jobserver")
|
||||
|
||||
# Phony install/* target should install picked package and all its deps
|
||||
specs_that_make_would_install = _parse_dry_run_package_installs(
|
||||
make("-n", picked_spec.format("install/{name}-{version}-{hash}"), output=str)
|
||||
)
|
||||
|
||||
assert set(specs_that_make_would_install) == set(expected_installs)
|
||||
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
|
||||
|
||||
# Phony install-deps/* target shouldn't install picked package
|
||||
specs_that_make_would_install = _parse_dry_run_package_installs(
|
||||
make("-n", picked_spec.format("install-deps/{name}-{version}-{hash}"), output=str)
|
||||
)
|
||||
|
||||
assert set(specs_that_make_would_install) == set(expected_installs) - {picked_package}
|
||||
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
|
||||
|
||||
|
||||
def test_environment_depfile_out(tmpdir, mock_packages):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user