zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract depfile logic from cli command into a core module (#36995)

Browse Source
This commit is contained in:
Harmen Stoppels
2023-04-19 14:36:29 +02:00
committed by GitHub
parent 3a5e48f476
commit ae909b3688
3 changed files with 316 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
lib/spack/spack/cmd/env.py
View File

@@ -4,7 +4,6 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import io
import os
import shutil
import sys
@@ -24,10 +23,11 @@
import spack.cmd.uninstall
import spack.config
import spack.environment as ev
import spack.environment.depfile as depfile
import spack.environment.shell
import spack.schema.env
import spack.spec
import spack.tengine
import spack.traverse as traverse
import spack.util.string as string
from spack.util.environment import EnvironmentModifications
@@ -637,161 +637,23 @@ def env_depfile_setup_parser(subparser):
)
def _deptypes(use_buildcache):
"""What edges should we follow for a given node? If it's a cache-only
node, then we can drop build type deps."""
return ("link", "run") if use_buildcache == "only" else ("build", "link", "run")
class MakeTargetVisitor(object):
"""This visitor produces an adjacency list of a (reduced) DAG, which
is used to generate Makefile targets with their prerequisites."""
def __init__(self, target, pkg_buildcache, deps_buildcache):
"""
Args:
target: function that maps dag_hash -> make target string
pkg_buildcache (str): "only", "never", "auto": when "only",
redundant build deps of roots are dropped
deps_buildcache (str): same as pkg_buildcache, but for non-root specs.
"""
self.adjacency_list = []
self.target = target
self.pkg_buildcache = pkg_buildcache
self.deps_buildcache = deps_buildcache
self.deptypes_root = _deptypes(pkg_buildcache)
self.deptypes_deps = _deptypes(deps_buildcache)
def neighbors(self, node):
"""Produce a list of spec to follow from node"""
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
def build_cache_flag(self, depth):
setting = self.pkg_buildcache if depth == 0 else self.deps_buildcache
if setting == "only":
return "--use-buildcache=only"
elif setting == "never":
return "--use-buildcache=never"
return ""
def accept(self, node):
fmt = "{name}-{version}-{hash}"
tgt = node.edge.spec.format(fmt)
spec_str = node.edge.spec.format(
"{name}{@version}{%compiler}{variants}{arch=architecture}"
)
buildcache_flag = self.build_cache_flag(node.depth)
prereqs = " ".join([self.target(dep.spec.format(fmt)) for dep in self.neighbors(node)])
self.adjacency_list.append(
(tgt, prereqs, node.edge.spec.dag_hash(), spec_str, buildcache_flag)
)
# We already accepted this
return True
def env_depfile(args):
# Currently only make is supported.
spack.cmd.require_active_env(cmd_name="env depfile")
env = ev.active_environment()
# Special make targets are useful when including a makefile in another, and you
# need to "namespace" the targets to avoid conflicts.
if args.make_prefix is None:
prefix = os.path.join(env.env_subdir_path, "makedeps")
else:
prefix = args.make_prefix
def get_target(name):
# The `all` and `clean` targets are phony. It doesn't make sense to
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
# sense to have a prefix like `env/all`, `env/clean` when they are
# supposed to be included
if name in ("all", "clean") and os.path.isabs(prefix):
return name
else:
return os.path.join(prefix, name)
def get_install_target(name):
return os.path.join(prefix, "install", name)
def get_install_deps_target(name):
return os.path.join(prefix, "install-deps", name)
# What things do we build when running make? By default, we build the
# root specs. If specific specs are provided as input, we build those.
if args.specs:
abstract_specs = spack.cmd.parse_specs(args.specs)
roots = [env.matching_spec(s) for s in abstract_specs]
else:
roots = [s for _, s in env.concretized_specs()]
# We produce a sub-DAG from the DAG induced by roots, where we drop build
# edges for those specs that are installed through a binary cache.
pkg_buildcache, dep_buildcache = args.use_buildcache
make_targets = MakeTargetVisitor(get_install_target, pkg_buildcache, dep_buildcache)
traverse.traverse_breadth_first_with_visitor(
roots, traverse.CoverNodesVisitor(make_targets, key=lambda s: s.dag_hash())
)
# Root specs without deps are the prereqs for the environment target
root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
all_pkg_identifiers = []
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
# generated makefiles to add post-install hooks, like pushing to a buildcache,
# running tests, etc.
# NOTE: GNU Make allows directory separators in variable names, so for consistency
# we can namespace this variable with the same prefix as targets.
if args.make_prefix is None:
pkg_identifier_variable = "SPACK_PACKAGE_IDS"
else:
pkg_identifier_variable = os.path.join(prefix, "SPACK_PACKAGE_IDS")
# All install and install-deps targets
all_install_related_targets = []
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
# we don't have a custom make target prefix.
phony_convenience_targets = []
for tgt, _, _, _, _ in make_targets.adjacency_list:
all_pkg_identifiers.append(tgt)
all_install_related_targets.append(get_install_target(tgt))
all_install_related_targets.append(get_install_deps_target(tgt))
if args.make_prefix is None:
phony_convenience_targets.append(os.path.join("install", tgt))
phony_convenience_targets.append(os.path.join("install-deps", tgt))
buf = io.StringIO()
filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None
template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
rendered = template.render(
{
"all_target": get_target("all"),
"env_target": get_target("env"),
"clean_target": get_target("clean"),
"all_install_related_targets": " ".join(all_install_related_targets),
"root_install_targets": " ".join(root_install_targets),
"dirs_target": get_target("dirs"),
"environment": env.path,
"install_target": get_target("install"),
"install_deps_target": get_target("install-deps"),
"any_hash_target": get_target("%"),
"jobserver_support": "+" if args.jobserver else "",
"adjacency_list": make_targets.adjacency_list,
"phony_convenience_targets": " ".join(phony_convenience_targets),
"pkg_ids_variable": pkg_identifier_variable,
"pkg_ids": " ".join(all_pkg_identifiers),
}
model = depfile.MakefileModel.from_env(
ev.active_environment(),
filter_specs=filter_specs,
pkg_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[0]),
dep_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[1]),
make_prefix=args.make_prefix,
jobserver=args.jobserver,
)
buf.write(rendered)
makefile = buf.getvalue()
makefile = template.render(model.to_dict())
# Finally write to stdout/file.
if args.output:

239
lib/spack/spack/environment/depfile.py Normal file
View File

@@ -0,0 +1,239 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
This module contains the traversal logic and models that can be used to generate
depfiles from an environment.
"""
import os
from enum import Enum
from typing import List, Optional
import spack.environment.environment as ev
import spack.spec
import spack.traverse as traverse
class UseBuildCache(Enum):
ONLY = 1
NEVER = 2
AUTO = 3
@staticmethod
def from_string(s: str) -> "UseBuildCache":
if s == "only":
return UseBuildCache.ONLY
elif s == "never":
return UseBuildCache.NEVER
elif s == "auto":
return UseBuildCache.AUTO
raise ValueError(f"invalid value for UseBuildCache: {s}")
def _deptypes(use_buildcache: UseBuildCache):
"""What edges should we follow for a given node? If it's a cache-only
node, then we can drop build type deps."""
return ("link", "run") if use_buildcache == UseBuildCache.ONLY else ("build", "link", "run")
class DepfileNode:
"""Contains a spec, a subset of its dependencies, and a flag whether it should be
buildcache only/never/auto."""
def __init__(
self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache
):
self.target = target
self.prereqs = prereqs
if buildcache == UseBuildCache.ONLY:
self.buildcache_flag = "--use-buildcache=only"
elif buildcache == UseBuildCache.NEVER:
self.buildcache_flag = "--use-buildcache=never"
else:
self.buildcache_flag = ""
class DepfileSpecVisitor:
"""This visitor produces an adjacency list of a (reduced) DAG, which
is used to generate depfile targets with their prerequisites. Currently
it only drops build deps when using buildcache only mode.
Note that the DAG could be reduced even more by dropping build edges of specs
installed at the moment the depfile is generated, but that would produce
stateful depfiles that would not fail when the database is wiped later."""
def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache):
self.adjacency_list: List[DepfileNode] = []
self.pkg_buildcache = pkg_buildcache
self.deps_buildcache = deps_buildcache
self.deptypes_root = _deptypes(pkg_buildcache)
self.deptypes_deps = _deptypes(deps_buildcache)
def neighbors(self, node):
"""Produce a list of spec to follow from node"""
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
def accept(self, node):
self.adjacency_list.append(
DepfileNode(
target=node.edge.spec,
prereqs=[edge.spec for edge in self.neighbors(node)],
buildcache=self.pkg_buildcache if node.depth == 0 else self.deps_buildcache,
)
)
# We already accepted this
return True
class MakefileModel:
"""This class produces all data to render a makefile for specs of an environment."""
def __init__(
self,
env: ev.Environment,
roots: List[spack.spec.Spec],
adjacency_list: List[DepfileNode],
make_prefix: Optional[str],
jobserver: bool,
):
"""
Args:
env: environment to generate the makefile for
roots: specs that get built in the default target
adjacency_list: list of DepfileNode, mapping specs to their dependencies
make_prefix: prefix for makefile targets
jobserver: when enabled, make will invoke Spack with jobserver support. For
dry-run this should be disabled.
"""
# Currently we can only use depfile with an environment since Spack needs to
# find the concrete specs somewhere.
self.env_path = env.path
# These specs are built in the default target.
self.roots = roots
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
# generated makefiles to add post-install hooks, like pushing to a buildcache,
# running tests, etc.
if make_prefix is None:
self.make_prefix = os.path.join(env.env_subdir_path, "makedeps")
self.pkg_identifier_variable = "SPACK_PACKAGE_IDS"
else:
# NOTE: GNU Make allows directory separators in variable names, so for consistency
# we can namespace this variable with the same prefix as targets.
self.make_prefix = make_prefix
self.pkg_identifier_variable = os.path.join(make_prefix, "SPACK_PACKAGE_IDS")
# And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag)
self.make_adjacency_list = [
(
self._safe_name(item.target),
" ".join(self._install_target(self._safe_name(s)) for s in item.prereqs),
item.target.dag_hash(),
item.target.format("{name}{@version}{%compiler}{variants}{arch=architecture}"),
item.buildcache_flag,
)
for item in adjacency_list
]
# Root specs without deps are the prereqs for the environment target
self.root_install_targets = [self._install_target(self._safe_name(s)) for s in roots]
self.jobserver_support = "+" if jobserver else ""
# All package identifiers, used to generate the SPACK_PACKAGE_IDS variable
self.all_pkg_identifiers: List[str] = []
# All install and install-deps targets
self.all_install_related_targets: List[str] = []
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
# we don't have a custom make target prefix.
self.phony_convenience_targets: List[str] = []
for node in adjacency_list:
tgt = self._safe_name(node.target)
self.all_pkg_identifiers.append(tgt)
self.all_install_related_targets.append(self._install_target(tgt))
self.all_install_related_targets.append(self._install_deps_target(tgt))
if make_prefix is None:
self.phony_convenience_targets.append(os.path.join("install", tgt))
self.phony_convenience_targets.append(os.path.join("install-deps", tgt))
def _safe_name(self, spec: spack.spec.Spec) -> str:
return spec.format("{name}-{version}-{hash}")
def _target(self, name: str) -> str:
# The `all` and `clean` targets are phony. It doesn't make sense to
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
# sense to have a prefix like `env/all`, `env/clean` when they are
# supposed to be included
if name in ("all", "clean") and os.path.isabs(self.make_prefix):
return name
else:
return os.path.join(self.make_prefix, name)
def _install_target(self, name: str) -> str:
return os.path.join(self.make_prefix, "install", name)
def _install_deps_target(self, name: str) -> str:
return os.path.join(self.make_prefix, "install-deps", name)
def to_dict(self):
return {
"all_target": self._target("all"),
"env_target": self._target("env"),
"clean_target": self._target("clean"),
"all_install_related_targets": " ".join(self.all_install_related_targets),
"root_install_targets": " ".join(self.root_install_targets),
"dirs_target": self._target("dirs"),
"environment": self.env_path,
"install_target": self._target("install"),
"install_deps_target": self._target("install-deps"),
"any_hash_target": self._target("%"),
"jobserver_support": self.jobserver_support,
"adjacency_list": self.make_adjacency_list,
"phony_convenience_targets": " ".join(self.phony_convenience_targets),
"pkg_ids_variable": self.pkg_identifier_variable,
"pkg_ids": " ".join(self.all_pkg_identifiers),
}
@staticmethod
def from_env(
env: ev.Environment,
*,
filter_specs: Optional[List[spack.spec.Spec]] = None,
pkg_buildcache: UseBuildCache = UseBuildCache.AUTO,
dep_buildcache: UseBuildCache = UseBuildCache.AUTO,
make_prefix: Optional[str] = None,
jobserver: bool = True,
) -> "MakefileModel":
"""Produces a MakefileModel from an environment and a list of specs.
Args:
env: the environment to use
filter_specs: if provided, only these specs will be built from the environment,
otherwise the environment roots are used.
pkg_buildcache: whether to only use the buildcache for top-level specs.
dep_buildcache: whether to only use the buildcache for non-top-level specs.
make_prefix: the prefix for the makefile targets
jobserver: when enabled, make will invoke Spack with jobserver support. For
dry-run this should be disabled.
"""
# If no specs are provided as a filter, build all the specs in the environment.
if filter_specs:
entrypoints = [env.matching_spec(s) for s in filter_specs]
else:
entrypoints = [s for _, s in env.concretized_specs()]
visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache)
traverse.traverse_breadth_first_with_visitor(
entrypoints, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash())
)
return MakefileModel(env, entrypoints, visitor.adjacency_list, make_prefix, jobserver)

72
lib/spack/spack/test/cmd/env.py
View File

@@ -3025,6 +3025,15 @@ def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_m
assert current_versions == expected_versions
def _parse_dry_run_package_installs(make_output):
"""Parse `spack install ... # <spec>` output from a make dry run."""
return [
Spec(line.split("# ")[1]).name
for line in make_output.splitlines()
if line.startswith("spack")
]
@pytest.mark.parametrize(
"depfile_flags,expected_installs",
[
@@ -3108,15 +3117,66 @@ def test_environment_depfile_makefile(depfile_flags, expected_installs, tmpdir,
# Do make dry run.
out = make("-n", "-f", makefile, output=str)
# Spack install commands are of the form "spack install ... # <spec>",
# so we just parse the spec again, for simplicity.
specs_that_make_would_install = [
Spec(line.split("# ")[1]).name for line in out.splitlines() if line.startswith("spack")
]
specs_that_make_would_install = _parse_dry_run_package_installs(out)
# Check that all specs are there (without duplicates)
assert set(specs_that_make_would_install) == set(expected_installs)
assert len(specs_that_make_would_install) == len(expected_installs)
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
@pytest.mark.parametrize(
"picked_package,expected_installs",
[
(
"dttop",
[
"dtbuild2",
"dtlink2",
"dtrun2",
"dtbuild1",
"dtlink4",
"dtlink3",
"dtlink1",
"dtlink5",
"dtbuild3",
"dtrun3",
"dtrun1",
"dttop",
],
),
("dtrun1", ["dtlink5", "dtbuild3", "dtrun3", "dtrun1"]),
],
)
def test_depfile_phony_convenience_targets(
picked_package, expected_installs: set, tmpdir, mock_packages
):
"""Check whether convenience targets "install/%" and "install-deps/%" are created for
each package if "--make-prefix" is absent."""
make = Executable("make")
with fs.working_dir(str(tmpdir)):
with ev.Environment("."):
add("dttop")
concretize()
with ev.Environment(".") as e:
picked_spec = e.matching_spec(picked_package)
env("depfile", "-o", "Makefile", "--make-disable-jobserver")
# Phony install/* target should install picked package and all its deps
specs_that_make_would_install = _parse_dry_run_package_installs(
make("-n", picked_spec.format("install/{name}-{version}-{hash}"), output=str)
)
assert set(specs_that_make_would_install) == set(expected_installs)
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
# Phony install-deps/* target shouldn't install picked package
specs_that_make_would_install = _parse_dry_run_package_installs(
make("-n", picked_spec.format("install-deps/{name}-{version}-{hash}"), output=str)
)
assert set(specs_that_make_would_install) == set(expected_installs) - {picked_package}
assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))
def test_environment_depfile_out(tmpdir, mock_packages):