refactor: Index dependency metadata by when spec

Part 1 of making all package metadata indexed by `when` condition. This
will allow us to handle all the dictionaries on `PackageBase` consistently.

Convert the current dependency dictionary structure from this:

    { name: { when_spec: [Dependency ...] } }

to this:

    { when_spec: { name: [Dependency ...] } }

On an M1 mac, this actually shaves 5% off the time it takes to load all
packages, I think because we're able to trade off lookups by spec key
for more lookups by name.
This commit is contained in:
Todd Gamblin 2023-06-19 18:47:58 -07:00
parent 92e08b160e
commit 6542c94cc1
13 changed files with 265 additions and 151 deletions

View File

@ -694,9 +694,7 @@ def _unknown_variants_in_directives(pkgs, error_cls):
) )
# Check "depends_on" directive # Check "depends_on" directive
for _, triggers in pkg_cls.dependencies.items(): for trigger in pkg_cls.dependencies:
triggers = list(triggers)
for trigger in list(triggers):
vrn = spack.spec.Spec(trigger) vrn = spack.spec.Spec(trigger)
errors.extend( errors.extend(
_analyze_variants_in_directive( _analyze_variants_in_directive(
@ -736,70 +734,60 @@ def _issues_in_depends_on_directive(pkgs, error_cls):
for pkg_name in pkgs: for pkg_name in pkgs:
pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
filename = spack.repo.PATH.filename_for_package_name(pkg_name) filename = spack.repo.PATH.filename_for_package_name(pkg_name)
for dependency_name, dependency_data in pkg_cls.dependencies.items():
for when, deps_by_name in pkg_cls.dependencies.items():
for dep_name, dep in deps_by_name.items():
# Check if there are nested dependencies declared. We don't want directives like: # Check if there are nested dependencies declared. We don't want directives like:
# #
# depends_on('foo+bar ^fee+baz') # depends_on('foo+bar ^fee+baz')
# #
# but we'd like to have two dependencies listed instead. # but we'd like to have two dependencies listed instead.
for when, dependency_edge in dependency_data.items(): nested_dependencies = dep.spec.dependencies()
dependency_spec = dependency_edge.spec
nested_dependencies = dependency_spec.dependencies()
if nested_dependencies: if nested_dependencies:
summary = ( summary = f"{pkg_name}: nested dependency declaration '{dep.spec}'"
f"{pkg_name}: invalid nested dependency " ndir = len(nested_dependencies) + 1
f"declaration '{str(dependency_spec)}'"
)
details = [ details = [
f"split depends_on('{str(dependency_spec)}', when='{str(when)}') " f"split depends_on('{dep.spec}', when='{when}') into {ndir} directives",
f"into {len(nested_dependencies) + 1} directives",
f"in {filename}",
]
errors.append(error_cls(summary=summary, details=details))
for s in (dependency_spec, when):
if s.virtual and s.variants:
summary = f"{pkg_name}: virtual dependency cannot have variants"
details = [
f"remove variants from '{str(s)}' in depends_on directive",
f"in {filename}", f"in {filename}",
] ]
errors.append(error_cls(summary=summary, details=details)) errors.append(error_cls(summary=summary, details=details))
# No need to analyze virtual packages # No need to analyze virtual packages
if spack.repo.PATH.is_virtual(dependency_name): if spack.repo.PATH.is_virtual(dep_name):
continue continue
# check for unknown dependencies
try: try:
dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dependency_name) dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name)
except spack.repo.UnknownPackageError: except spack.repo.UnknownPackageError:
# This dependency is completely missing, so report # This dependency is completely missing, so report
# and continue the analysis # and continue the analysis
summary = pkg_name + ": unknown package '{0}' in " "'depends_on' directive".format( summary = (
dependency_name f"{pkg_name}: unknown package '{dep_name}' in " "'depends_on' directive"
) )
details = [" in " + filename] details = [f" in {filename}"]
errors.append(error_cls(summary=summary, details=details)) errors.append(error_cls(summary=summary, details=details))
continue continue
for _, dependency_edge in dependency_data.items(): # check variants
dependency_variants = dependency_edge.spec.variants dependency_variants = dep.spec.variants
for name, value in dependency_variants.items(): for name, value in dependency_variants.items():
try: try:
v, _ = dependency_pkg_cls.variants[name] v, _ = dependency_pkg_cls.variants[name]
v.validate_or_raise(value, pkg_cls=dependency_pkg_cls) v.validate_or_raise(value, pkg_cls=dependency_pkg_cls)
except Exception as e: except Exception as e:
summary = ( summary = (
pkg_name + ": wrong variant used for a " f"{pkg_name}: wrong variant used for dependency in 'depends_on()'"
"dependency in a 'depends_on' directive"
) )
error_msg = str(e).strip()
if isinstance(e, KeyError): if isinstance(e, KeyError):
error_msg = "the variant {0} does not " "exist".format(error_msg) error_msg = (
error_msg += " in package '" + dependency_name + "'" f"variant {str(e).strip()} does not exist in package {dep_name}"
)
error_msg += f" in package '{dep_name}'"
errors.append( errors.append(
error_cls(summary=summary, details=[error_msg, "in " + filename]) error_cls(summary=summary, details=[error_msg, f"in {filename}"])
) )
return errors return errors
@ -866,14 +854,17 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls
for pkg_name in pkgs: for pkg_name in pkgs:
pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
filename = spack.repo.PATH.filename_for_package_name(pkg_name) filename = spack.repo.PATH.filename_for_package_name(pkg_name)
dependencies_to_check = [] dependencies_to_check = []
for dependency_name, dependency_data in pkg_cls.dependencies.items():
for _, deps_by_name in pkg_cls.dependencies.items():
for dep_name, dep in deps_by_name.items():
# Skip virtual dependencies for the time being, check on # Skip virtual dependencies for the time being, check on
# their versions can be added later # their versions can be added later
if spack.repo.PATH.is_virtual(dependency_name): if spack.repo.PATH.is_virtual(dep_name):
continue continue
dependencies_to_check.extend([edge.spec for edge in dependency_data.values()]) dependencies_to_check.append(dep.spec)
host_architecture = spack.spec.ArchSpec.default_arch() host_architecture = spack.spec.ArchSpec.default_arch()
for s in dependencies_to_check: for s in dependencies_to_check:
@ -945,18 +936,28 @@ def _named_specs_in_when_arguments(pkgs, error_cls):
for pkg_name in pkgs: for pkg_name in pkgs:
pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
def _refers_to_pkg(when):
when_spec = spack.spec.Spec(when)
return when_spec.name is None or when_spec.name == pkg_name
def _error_items(when_dict):
for when, elts in when_dict.items():
if not _refers_to_pkg(when):
yield when, elts, [f"using '{when}', should be '^{when}'"]
def _extracts_errors(triggers, summary): def _extracts_errors(triggers, summary):
_errors = [] _errors = []
for trigger in list(triggers): for trigger in list(triggers):
when_spec = spack.spec.Spec(trigger) if not _refers_to_pkg(trigger):
if when_spec.name is not None and when_spec.name != pkg_name:
details = [f"using '{trigger}', should be '^{trigger}'"] details = [f"using '{trigger}', should be '^{trigger}'"]
_errors.append(error_cls(summary=summary, details=details)) _errors.append(error_cls(summary=summary, details=details))
return _errors return _errors
for dname, triggers in pkg_cls.dependencies.items(): for when, dnames, details in _error_items(pkg_cls.dependencies):
summary = f"{pkg_name}: wrong 'when=' condition for the '{dname}' dependency" errors.extend(
errors.extend(_extracts_errors(triggers, summary)) error_cls(f"{pkg_name}: wrong 'when=' condition for '{dname}' dependency", details)
for dname in dnames
)
for vname, (variant, triggers) in pkg_cls.variants.items(): for vname, (variant, triggers) in pkg_cls.variants.items():
summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant" summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant"
@ -971,13 +972,15 @@ def _extracts_errors(triggers, summary):
summary = f"{pkg_name}: wrong 'when=' condition in 'requires' directive" summary = f"{pkg_name}: wrong 'when=' condition in 'requires' directive"
errors.extend(_extracts_errors(triggers, summary)) errors.extend(_extracts_errors(triggers, summary))
triggers = list(pkg_cls.patches) for when, _, details in _error_items(pkg_cls.patches):
summary = f"{pkg_name}: wrong 'when=' condition in 'patch' directives" errors.append(
errors.extend(_extracts_errors(triggers, summary)) error_cls(f"{pkg_name}: wrong 'when=' condition in 'patch' directives", details)
)
triggers = list(pkg_cls.resources) for when, _, details in _error_items(pkg_cls.resources):
summary = f"{pkg_name}: wrong 'when=' condition in 'resource' directives" errors.append(
errors.extend(_extracts_errors(triggers, summary)) error_cls(f"{pkg_name}: wrong 'when=' condition in 'resource' directives", details)
)
return llnl.util.lang.dedupe(errors) return llnl.util.lang.dedupe(errors)

View File

@ -3,6 +3,7 @@
# #
# SPDX-License-Identifier: (Apache-2.0 OR MIT) # SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import sys import sys
import llnl.util.tty as tty import llnl.util.tty as tty
@ -49,7 +50,17 @@ def inverted_dependencies():
dag = {} dag = {}
for pkg_cls in spack.repo.PATH.all_package_classes(): for pkg_cls in spack.repo.PATH.all_package_classes():
dag.setdefault(pkg_cls.name, set()) dag.setdefault(pkg_cls.name, set())
for dep in pkg_cls.dependencies: for dep in pkg_cls.dependencies_by_name():
deps = [dep]
# expand virtuals if necessary
if spack.repo.PATH.is_virtual(dep):
deps += [s.name for s in spack.repo.PATH.providers_for(dep)]
dag = collections.defaultdict(set)
for pkg_cls in spack.repo.PATH.all_package_classes():
for _, deps_by_name in pkg_cls.dependencies.items():
for dep in deps_by_name:
deps = [dep] deps = [dep]
# expand virtuals if necessary # expand virtuals if necessary
@ -57,7 +68,7 @@ def inverted_dependencies():
deps += [s.name for s in spack.repo.PATH.providers_for(dep)] deps += [s.name for s in spack.repo.PATH.providers_for(dep)]
for d in deps: for d in deps:
dag.setdefault(d, set()).add(pkg_cls.name) dag[d].add(pkg_cls.name)
return dag return dag

View File

@ -79,4 +79,7 @@ def merge(self, other: "Dependency"):
def __repr__(self) -> str: def __repr__(self) -> str:
types = dt.flag_to_chars(self.depflag) types = dt.flag_to_chars(self.depflag)
if self.patches:
return f"<Dependency: {self.pkg.name} -> {self.spec} [{types}, {self.patches}]>"
else:
return f"<Dependency: {self.pkg.name} -> {self.spec} [{types}]>" return f"<Dependency: {self.pkg.name} -> {self.spec} [{types}]>"

View File

@ -29,6 +29,7 @@ class OpenMpi(Package):
* ``requires`` * ``requires``
""" """
import collections
import collections.abc import collections.abc
import functools import functools
import os.path import os.path
@ -83,7 +84,14 @@ class OpenMpi(Package):
_patch_order_index = 0 _patch_order_index = 0
def make_when_spec(value): SpecType = Union["spack.spec.Spec", str]
DepType = Union[Tuple[str, ...], str]
WhenType = Optional[Union["spack.spec.Spec", str, bool]]
Patcher = Callable[[Union["spack.package_base.PackageBase", Dependency]], None]
PatchesType = Optional[Union[Patcher, str, List[Union[Patcher, str]]]]
def make_when_spec(value: WhenType) -> Optional["spack.spec.Spec"]:
"""Create a ``Spec`` that indicates when a directive should be applied. """Create a ``Spec`` that indicates when a directive should be applied.
Directives with ``when`` specs, e.g.: Directives with ``when`` specs, e.g.:
@ -106,7 +114,7 @@ def make_when_spec(value):
as part of concretization. as part of concretization.
Arguments: Arguments:
value (spack.spec.Spec or bool): a conditional Spec or a constant ``bool`` value: a conditional Spec, constant ``bool``, or None if not supplied
value indicating when a directive should be applied. value indicating when a directive should be applied.
""" """
@ -116,7 +124,7 @@ def make_when_spec(value):
# Unsatisfiable conditions are discarded by the caller, and never # Unsatisfiable conditions are discarded by the caller, and never
# added to the package class # added to the package class
if value is False: if value is False:
return False return None
# If there is no constraint, the directive should always apply; # If there is no constraint, the directive should always apply;
# represent this by returning the unconstrained `Spec()`, which is # represent this by returning the unconstrained `Spec()`, which is
@ -175,8 +183,8 @@ def __init__(cls, name, bases, attr_dict):
# that the directives are called to set it up. # that the directives are called to set it up.
if "spack.pkg" in cls.__module__: if "spack.pkg" in cls.__module__:
# Ensure the presence of the dictionaries associated # Ensure the presence of the dictionaries associated with the directives.
# with the directives # All dictionaries are defaultdicts that create lists for missing keys.
for d in DirectiveMeta._directive_dict_names: for d in DirectiveMeta._directive_dict_names:
setattr(cls, d, {}) setattr(cls, d, {})
@ -455,7 +463,14 @@ def _execute_version(pkg, ver, **kwargs):
pkg.versions[version] = kwargs pkg.versions[version] = kwargs
def _depends_on(pkg, spec, when=None, type=dt.DEFAULT_TYPES, patches=None): def _depends_on(
pkg: "spack.package_base.PackageBase",
spec: SpecType,
*,
when: WhenType = None,
type: DepType = dt.DEFAULT_TYPES,
patches: PatchesType = None,
):
when_spec = make_when_spec(when) when_spec = make_when_spec(when)
if not when_spec: if not when_spec:
return return
@ -467,7 +482,6 @@ def _depends_on(pkg, spec, when=None, type=dt.DEFAULT_TYPES, patches=None):
raise CircularReferenceError("Package '%s' cannot depend on itself." % pkg.name) raise CircularReferenceError("Package '%s' cannot depend on itself." % pkg.name)
depflag = dt.canonicalize(type) depflag = dt.canonicalize(type)
conditions = pkg.dependencies.setdefault(dep_spec.name, {})
# call this patches here for clarity -- we want patch to be a list, # call this patches here for clarity -- we want patch to be a list,
# but the caller doesn't have to make it one. # but the caller doesn't have to make it one.
@ -495,11 +509,13 @@ def _depends_on(pkg, spec, when=None, type=dt.DEFAULT_TYPES, patches=None):
assert all(callable(p) for p in patches) assert all(callable(p) for p in patches)
# this is where we actually add the dependency to this package # this is where we actually add the dependency to this package
if when_spec not in conditions: deps_by_name = pkg.dependencies.setdefault(when_spec, {})
dependency = deps_by_name.get(dep_spec.name)
if not dependency:
dependency = Dependency(pkg, dep_spec, depflag=depflag) dependency = Dependency(pkg, dep_spec, depflag=depflag)
conditions[when_spec] = dependency deps_by_name[dep_spec.name] = dependency
else: else:
dependency = conditions[when_spec]
dependency.spec.constrain(dep_spec, deps=False) dependency.spec.constrain(dep_spec, deps=False)
dependency.depflag |= depflag dependency.depflag |= depflag
@ -544,15 +560,20 @@ def _execute_conflicts(pkg):
@directive(("dependencies")) @directive(("dependencies"))
def depends_on(spec, when=None, type=dt.DEFAULT_TYPES, patches=None): def depends_on(
spec: SpecType,
when: WhenType = None,
type: DepType = dt.DEFAULT_TYPES,
patches: PatchesType = None,
):
"""Creates a dict of deps with specs defining when they apply. """Creates a dict of deps with specs defining when they apply.
Args: Args:
spec (spack.spec.Spec or str): the package and constraints depended on spec: the package and constraints depended on
when (spack.spec.Spec or str): when the dependent satisfies this, it has when: when the dependent satisfies this, it has
the dependency represented by ``spec`` the dependency represented by ``spec``
type (str or tuple): str or tuple of legal Spack deptypes type: str or tuple of legal Spack deptypes
patches (typing.Callable or list): single result of ``patch()`` directive, a patches: single result of ``patch()`` directive, a
``str`` to be passed to ``patch``, or a list of these ``str`` to be passed to ``patch``, or a list of these
This directive is to be used inside a Package definition to declare This directive is to be used inside a Package definition to declare
@ -561,7 +582,7 @@ def depends_on(spec, when=None, type=dt.DEFAULT_TYPES, patches=None):
""" """
def _execute_depends_on(pkg): def _execute_depends_on(pkg: "spack.package_base.PackageBase"):
_depends_on(pkg, spec, when=when, type=type, patches=patches) _depends_on(pkg, spec, when=when, type=type, patches=patches)
return _execute_depends_on return _execute_depends_on
@ -630,28 +651,31 @@ def _execute_provides(pkg):
@directive("patches") @directive("patches")
def patch(url_or_filename, level=1, when=None, working_dir=".", **kwargs): def patch(
url_or_filename: str,
level: int = 1,
when: WhenType = None,
working_dir: str = ".",
sha256: Optional[str] = None,
archive_sha256: Optional[str] = None,
) -> Patcher:
"""Packages can declare patches to apply to source. You can """Packages can declare patches to apply to source. You can
optionally provide a when spec to indicate that a particular optionally provide a when spec to indicate that a particular
patch should only be applied when the package's spec meets patch should only be applied when the package's spec meets
certain conditions (e.g. a particular version). certain conditions (e.g. a particular version).
Args: Args:
url_or_filename (str): url or relative filename of the patch url_or_filename: url or relative filename of the patch
level (int): patch level (as in the patch shell command) level: patch level (as in the patch shell command)
when (spack.spec.Spec): optional anonymous spec that specifies when to apply when: optional anonymous spec that specifies when to apply the patch
the patch working_dir: dir to change to before applying
working_dir (str): dir to change to before applying sha256: sha256 sum of the patch, used to verify the patch (only required for URL patches)
archive_sha256: sha256 sum of the *archive*, if the patch is compressed (only required for
Keyword Args: compressed URL patches)
sha256 (str): sha256 sum of the patch, used to verify the patch
(only required for URL patches)
archive_sha256 (str): sha256 sum of the *archive*, if the patch
is compressed (only required for compressed URL patches)
""" """
def _execute_patch(pkg_or_dep): def _execute_patch(pkg_or_dep: Union["spack.package_base.PackageBase", Dependency]):
pkg = pkg_or_dep pkg = pkg_or_dep
if isinstance(pkg, Dependency): if isinstance(pkg, Dependency):
pkg = pkg.pkg pkg = pkg.pkg
@ -673,9 +697,16 @@ def _execute_patch(pkg_or_dep):
ordering_key = (pkg.name, _patch_order_index) ordering_key = (pkg.name, _patch_order_index)
_patch_order_index += 1 _patch_order_index += 1
patch: spack.patch.Patch
if "://" in url_or_filename: if "://" in url_or_filename:
patch = spack.patch.UrlPatch( patch = spack.patch.UrlPatch(
pkg, url_or_filename, level, working_dir, ordering_key=ordering_key, **kwargs pkg,
url_or_filename,
level,
working_dir,
ordering_key=ordering_key,
sha256=sha256,
archive_sha256=archive_sha256,
) )
else: else:
patch = spack.patch.FilePatch( patch = spack.patch.FilePatch(

View File

@ -25,7 +25,7 @@
import time import time
import traceback import traceback
import warnings import warnings
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, TypeVar from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union
import llnl.util.filesystem as fsys import llnl.util.filesystem as fsys
import llnl.util.tty as tty import llnl.util.tty as tty
@ -432,6 +432,43 @@ def remove_files_from_view(self, view, merge_map):
Pb = TypeVar("Pb", bound="PackageBase") Pb = TypeVar("Pb", bound="PackageBase")
WhenDict = Dict[spack.spec.Spec, Dict[str, Any]]
NameValuesDict = Dict[str, List[Any]]
NameWhenDict = Dict[str, Dict[spack.spec.Spec, List[Any]]]
def _by_name(
when_indexed_dictionary: WhenDict, when: bool = False
) -> Union[NameValuesDict, NameWhenDict]:
"""Convert a dict of dicts keyed by when/name into a dict of lists keyed by name.
Optional Arguments:
when: if ``True``, don't discared the ``when`` specs; return a 2-level dictionary
keyed by name and when spec.
"""
# very hard to define this type to be conditional on `when`
all_by_name: Dict[str, Any] = {}
for when_spec, by_name in when_indexed_dictionary.items():
for name, value in by_name.items():
if when:
when_dict = all_by_name.setdefault(name, {})
when_dict.setdefault(when_spec, []).append(value)
else:
all_by_name.setdefault(name, []).append(value)
return dict(sorted(all_by_name.items()))
def _names(when_indexed_dictionary):
"""Get sorted names from dicts keyed by when/name."""
all_names = set()
for when, by_name in when_indexed_dictionary.items():
for name in by_name:
all_names.add(name)
return sorted(all_names)
class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
"""This is the superclass for all spack packages. """This is the superclass for all spack packages.
@ -525,7 +562,10 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
versions: dict versions: dict
# Same for dependencies # Same for dependencies
dependencies: dict dependencies: Dict["spack.spec.Spec", Dict[str, "spack.dependency.Dependency"]]
# and patches
patches: Dict["spack.spec.Spec", List["spack.patch.Patch"]]
#: By default, packages are not virtual #: By default, packages are not virtual
#: Virtual packages override this attribute #: Virtual packages override this attribute
@ -679,6 +719,14 @@ def __init__(self, spec):
super().__init__() super().__init__()
@classmethod
def dependency_names(cls):
return _names(cls.dependencies)
@classmethod
def dependencies_by_name(cls, when: bool = False):
return _by_name(cls.dependencies, when=when)
@classmethod @classmethod
def possible_dependencies( def possible_dependencies(
cls, cls,
@ -727,10 +775,11 @@ def possible_dependencies(
visited.setdefault(cls.name, set()) visited.setdefault(cls.name, set())
for name, conditions in cls.dependencies.items(): for name, conditions in cls.dependencies_by_name(when=True).items():
# check whether this dependency could be of the type asked for # check whether this dependency could be of the type asked for
depflag_union = 0 depflag_union = 0
for dep in conditions.values(): for deplist in conditions.values():
for dep in deplist:
depflag_union |= dep.depflag depflag_union |= dep.depflag
if not (depflag & depflag_union): if not (depflag & depflag_union):
continue continue
@ -1219,7 +1268,7 @@ def fetcher(self, f):
@classmethod @classmethod
def dependencies_of_type(cls, deptypes: dt.DepFlag): def dependencies_of_type(cls, deptypes: dt.DepFlag):
"""Get dependencies that can possibly have these deptypes. """Get names of dependencies that can possibly have these deptypes.
This analyzes the package and determines which dependencies *can* This analyzes the package and determines which dependencies *can*
be a certain kind of dependency. Note that they may not *always* be a certain kind of dependency. Note that they may not *always*
@ -1227,11 +1276,11 @@ def dependencies_of_type(cls, deptypes: dt.DepFlag):
so something may be a build dependency in one configuration and a so something may be a build dependency in one configuration and a
run dependency in another. run dependency in another.
""" """
return dict( return {
(name, conds) name
for name, conds in cls.dependencies.items() for name, dependencies in cls.dependencies_by_name().items()
if any(deptypes & cls.dependencies[name][cond].depflag for cond in conds) if any(deptypes & dep.depflag for dep in dependencies)
) }
# TODO: allow more than one active extendee. # TODO: allow more than one active extendee.
@property @property

View File

@ -389,10 +389,9 @@ def _index_patches(pkg_class, repository):
patch_dict.pop("sha256") # save some space patch_dict.pop("sha256") # save some space
index[patch.sha256] = {pkg_class.fullname: patch_dict} index[patch.sha256] = {pkg_class.fullname: patch_dict}
# and patches on dependencies for deps_by_name in pkg_class.dependencies.values():
for name, conditions in pkg_class.dependencies.items(): for dependency in deps_by_name.values():
for cond, dependency in conditions.items(): for patch_list in dependency.patches.values():
for pcond, patch_list in dependency.patches.items():
for patch in patch_list: for patch in patch_list:
dspec_cls = repository.get_pkg_class(dependency.spec.name) dspec_cls = repository.get_pkg_class(dependency.spec.name)
patch_dict = patch.to_dict() patch_dict = patch.to_dict()

View File

@ -1643,8 +1643,8 @@ def package_provider_rules(self, pkg):
def package_dependencies_rules(self, pkg): def package_dependencies_rules(self, pkg):
"""Translate 'depends_on' directives into ASP logic.""" """Translate 'depends_on' directives into ASP logic."""
for _, conditions in sorted(pkg.dependencies.items()): for cond, deps_by_name in sorted(pkg.dependencies.items()):
for cond, dep in sorted(conditions.items()): for _, dep in sorted(deps_by_name.items()):
depflag = dep.depflag depflag = dep.depflag
# Skip test dependencies if they're not requested # Skip test dependencies if they're not requested
if not self.tests: if not self.tests:
@ -1741,6 +1741,7 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):
pkg_name, policy, requirement_grp = rule.pkg_name, rule.policy, rule.requirements pkg_name, policy, requirement_grp = rule.pkg_name, rule.policy, rule.requirements
requirement_weight = 0 requirement_weight = 0
# TODO: don't call make_when_spec here; do it in directives.
main_requirement_condition = spack.directives.make_when_spec(rule.condition) main_requirement_condition = spack.directives.make_when_spec(rule.condition)
if main_requirement_condition is False: if main_requirement_condition is False:
continue continue
@ -1750,7 +1751,7 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):
msg = f"condition to activate requirement {requirement_grp_id}" msg = f"condition to activate requirement {requirement_grp_id}"
try: try:
main_condition_id = self.condition( main_condition_id = self.condition(
main_requirement_condition, name=pkg_name, msg=msg main_requirement_condition, name=pkg_name, msg=msg # type: ignore
) )
except Exception as e: except Exception as e:
if rule.kind != RequirementKind.DEFAULT: if rule.kind != RequirementKind.DEFAULT:

View File

@ -2876,13 +2876,14 @@ def inject_patches_variant(root):
continue continue
# Add any patches from the package to the spec. # Add any patches from the package to the spec.
patches = [] patches = set()
for cond, patch_list in s.package_class.patches.items(): for cond, patch_list in s.package_class.patches.items():
if s.satisfies(cond): if s.satisfies(cond):
for patch in patch_list: for patch in patch_list:
patches.append(patch) patches.add(patch)
if patches: if patches:
spec_to_patches[id(s)] = patches spec_to_patches[id(s)] = patches
# Also record all patches required on dependencies by # Also record all patches required on dependencies by
# depends_on(..., patch=...) # depends_on(..., patch=...)
for dspec in root.traverse_edges(deptype=all, cover="edges", root=False): for dspec in root.traverse_edges(deptype=all, cover="edges", root=False):
@ -2890,17 +2891,25 @@ def inject_patches_variant(root):
continue continue
pkg_deps = dspec.parent.package_class.dependencies pkg_deps = dspec.parent.package_class.dependencies
if dspec.spec.name not in pkg_deps:
continue
patches = [] patches = []
for cond, dependency in pkg_deps[dspec.spec.name].items(): for cond, deps_by_name in pkg_deps.items():
if not dspec.parent.satisfies(cond):
continue
dependency = deps_by_name.get(dspec.spec.name)
if not dependency:
continue
for pcond, patch_list in dependency.patches.items(): for pcond, patch_list in dependency.patches.items():
if dspec.parent.satisfies(cond) and dspec.spec.satisfies(pcond): if dspec.spec.satisfies(pcond):
patches.extend(patch_list) patches.extend(patch_list)
if patches: if patches:
all_patches = spec_to_patches.setdefault(id(dspec.spec), []) all_patches = spec_to_patches.setdefault(id(dspec.spec), set())
all_patches.extend(patches) for patch in patches:
all_patches.add(patch)
for spec in root.traverse(): for spec in root.traverse():
if id(spec) not in spec_to_patches: if id(spec) not in spec_to_patches:
continue continue
@ -3163,13 +3172,17 @@ def _evaluate_dependency_conditions(self, name):
If no conditions are True (and we don't depend on it), return If no conditions are True (and we don't depend on it), return
``(None, None)``. ``(None, None)``.
""" """
conditions = self.package_class.dependencies[name]
vt.substitute_abstract_variants(self) vt.substitute_abstract_variants(self)
# evaluate when specs to figure out constraints on the dependency. # evaluate when specs to figure out constraints on the dependency.
dep = None dep = None
for when_spec, dependency in conditions.items(): for when_spec, deps_by_name in self.package_class.dependencies.items():
if self.satisfies(when_spec): if not self.satisfies(when_spec):
continue
for dep_name, dependency in deps_by_name.items():
if dep_name != name:
continue
if dep is None: if dep is None:
dep = dp.Dependency(Spec(self.name), Spec(name), depflag=0) dep = dp.Dependency(Spec(self.name), Spec(name), depflag=0)
try: try:
@ -3344,7 +3357,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index, tests):
while changed: while changed:
changed = False changed = False
for dep_name in self.package_class.dependencies: for dep_name in self.package_class.dependency_names():
# Do we depend on dep_name? If so pkg_dep is not None. # Do we depend on dep_name? If so pkg_dep is not None.
dep = self._evaluate_dependency_conditions(dep_name) dep = self._evaluate_dependency_conditions(dep_name)

View File

@ -29,8 +29,8 @@ def test_true_directives_exist(mock_packages):
cls = spack.repo.PATH.get_pkg_class("when-directives-true") cls = spack.repo.PATH.get_pkg_class("when-directives-true")
assert cls.dependencies assert cls.dependencies
assert spack.spec.Spec() in cls.dependencies["extendee"] assert "extendee" in cls.dependencies[spack.spec.Spec()]
assert spack.spec.Spec() in cls.dependencies["b"] assert "b" in cls.dependencies[spack.spec.Spec()]
assert cls.resources assert cls.resources
assert spack.spec.Spec() in cls.resources assert spack.spec.Spec() in cls.resources
@ -43,7 +43,7 @@ def test_constraints_from_context(mock_packages):
pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met")
assert pkg_cls.dependencies assert pkg_cls.dependencies
assert spack.spec.Spec("@1.0") in pkg_cls.dependencies["b"] assert "b" in pkg_cls.dependencies[spack.spec.Spec("@1.0")]
assert pkg_cls.conflicts assert pkg_cls.conflicts
assert (spack.spec.Spec("+foo@1.0"), None) in pkg_cls.conflicts["%gcc"] assert (spack.spec.Spec("+foo@1.0"), None) in pkg_cls.conflicts["%gcc"]
@ -54,7 +54,7 @@ def test_constraints_from_context_are_merged(mock_packages):
pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met")
assert pkg_cls.dependencies assert pkg_cls.dependencies
assert spack.spec.Spec("@0.14:15 ^b@3.8:4.0") in pkg_cls.dependencies["c"] assert "c" in pkg_cls.dependencies[spack.spec.Spec("@0.14:15 ^b@3.8:4.0")]
@pytest.mark.regression("27754") @pytest.mark.regression("27754")

View File

@ -71,7 +71,8 @@ def test_possible_direct_dependencies(mock_packages, mpileaks_possible_deps):
def test_possible_dependencies_virtual(mock_packages, mpi_names): def test_possible_dependencies_virtual(mock_packages, mpi_names):
expected = dict( expected = dict(
(name, set(spack.repo.PATH.get_pkg_class(name).dependencies)) for name in mpi_names (name, set(dep for dep in spack.repo.PATH.get_pkg_class(name).dependencies_by_name()))
for name in mpi_names
) )
# only one mock MPI has a dependency # only one mock MPI has a dependency

View File

@ -61,14 +61,15 @@ def test_import_package_as(self):
import spack.pkg.builtin.mock.mpich as mp # noqa: F401 import spack.pkg.builtin.mock.mpich as mp # noqa: F401
from spack.pkg.builtin import mock # noqa: F401 from spack.pkg.builtin import mock # noqa: F401
def test_inheritance_of_diretives(self): def test_inheritance_of_directives(self):
pkg_cls = spack.repo.PATH.get_pkg_class("simple-inheritance") pkg_cls = spack.repo.PATH.get_pkg_class("simple-inheritance")
# Check dictionaries that should have been filled by directives # Check dictionaries that should have been filled by directives
assert len(pkg_cls.dependencies) == 3 dependencies = pkg_cls.dependencies_by_name()
assert "cmake" in pkg_cls.dependencies assert len(dependencies) == 3
assert "openblas" in pkg_cls.dependencies assert "cmake" in dependencies
assert "mpi" in pkg_cls.dependencies assert "openblas" in dependencies
assert "mpi" in dependencies
assert len(pkg_cls.provided) == 2 assert len(pkg_cls.provided) == 2
# Check that Spec instantiation behaves as we expect # Check that Spec instantiation behaves as we expect

View File

@ -196,16 +196,18 @@ def test_nested_directives(mock_packages):
# this ensures that results of dependency patches were properly added # this ensures that results of dependency patches were properly added
# to Dependency objects. # to Dependency objects.
libelf_dep = next(iter(patcher.dependencies["libelf"].values())) deps_by_name = patcher.dependencies_by_name()
libelf_dep = deps_by_name["libelf"][0]
assert len(libelf_dep.patches) == 1 assert len(libelf_dep.patches) == 1
assert len(libelf_dep.patches[Spec()]) == 1 assert len(libelf_dep.patches[Spec()]) == 1
libdwarf_dep = next(iter(patcher.dependencies["libdwarf"].values())) libdwarf_dep = deps_by_name["libdwarf"][0]
assert len(libdwarf_dep.patches) == 2 assert len(libdwarf_dep.patches) == 2
assert len(libdwarf_dep.patches[Spec()]) == 1 assert len(libdwarf_dep.patches[Spec()]) == 1
assert len(libdwarf_dep.patches[Spec("@20111030")]) == 1 assert len(libdwarf_dep.patches[Spec("@20111030")]) == 1
fake_dep = next(iter(patcher.dependencies["fake"].values())) fake_dep = deps_by_name["fake"][0]
assert len(fake_dep.patches) == 1 assert len(fake_dep.patches) == 1
assert len(fake_dep.patches[Spec()]) == 2 assert len(fake_dep.patches[Spec()]) == 2

View File

@ -51,7 +51,7 @@ def _mock(pkg_name, spec):
cond = Spec(pkg_cls.name) cond = Spec(pkg_cls.name)
dependency = Dependency(pkg_cls, spec) dependency = Dependency(pkg_cls, spec)
monkeypatch.setitem(pkg_cls.dependencies, spec.name, {cond: dependency}) monkeypatch.setitem(pkg_cls.dependencies, cond, {spec.name: dependency})
return _mock return _mock