solver: partition classes related to requirement parsing into their own file (#47915)
This commit is contained in:
Massimiliano Culpo
GitHub
parent
4693b323ac
commit
c1b2ac549d
@ -48,8 +48,6 @@
|
||||
import spack.version as vn
|
||||
import spack.version.git_ref_lookup
|
||||
from spack import traverse
|
||||
from spack.config import get_mark_from_yaml_data
|
||||
from spack.error import SpecSyntaxError
|
||||
|
||||
from .core import (
|
||||
AspFunction,
|
||||
@ -65,6 +63,7 @@
|
||||
parse_term,
|
||||
)
|
||||
from .counter import FullDuplicatesCounter, MinimalDuplicatesCounter, NoDuplicatesCounter
|
||||
from .requirements import RequirementKind, RequirementParser, RequirementRule
|
||||
from .version_order import concretization_version_order
|
||||
|
||||
GitOrStandardVersion = Union[spack.version.GitVersion, spack.version.StandardVersion]
|
||||
@ -144,17 +143,6 @@ def named_spec(
|
||||
spec.name = old_name
|
||||
|
||||
|
||||
class RequirementKind(enum.Enum):
|
||||
"""Purpose / provenance of a requirement"""
|
||||
|
||||
#: Default requirement expressed under the 'all' attribute of packages.yaml
|
||||
DEFAULT = enum.auto()
|
||||
#: Requirement expressed on a virtual package
|
||||
VIRTUAL = enum.auto()
|
||||
#: Requirement expressed on a specific package
|
||||
PACKAGE = enum.auto()
|
||||
|
||||
|
||||
class DeclaredVersion(NamedTuple):
|
||||
"""Data class to contain information on declared versions used in the solve"""
|
||||
|
||||
@ -757,17 +745,6 @@ def on_model(model):
|
||||
raise UnsatisfiableSpecError(msg)
|
||||
|
||||
|
||||
class RequirementRule(NamedTuple):
|
||||
"""Data class to collect information on a requirement"""
|
||||
|
||||
pkg_name: str
|
||||
policy: str
|
||||
requirements: List["spack.spec.Spec"]
|
||||
condition: "spack.spec.Spec"
|
||||
kind: RequirementKind
|
||||
message: Optional[str]
|
||||
|
||||
|
||||
class KnownCompiler(NamedTuple):
|
||||
"""Data class to collect information on compilers"""
|
||||
|
||||
@ -1146,6 +1123,7 @@ class SpackSolverSetup:
|
||||
def __init__(self, tests: bool = False):
|
||||
# these are all initialized in setup()
|
||||
self.gen: "ProblemInstanceBuilder" = ProblemInstanceBuilder()
|
||||
self.requirement_parser = RequirementParser(spack.config.CONFIG)
|
||||
self.possible_virtuals: Set[str] = set()
|
||||
|
||||
self.assumptions: List[Tuple["clingo.Symbol", bool]] = [] # type: ignore[name-defined]
|
||||
@ -1332,8 +1310,7 @@ def compiler_facts(self):
|
||||
self.gen.newline()
|
||||
|
||||
def package_requirement_rules(self, pkg):
|
||||
parser = RequirementParser(spack.config.CONFIG)
|
||||
self.emit_facts_from_requirement_rules(parser.rules(pkg))
|
||||
self.emit_facts_from_requirement_rules(self.requirement_parser.rules(pkg))
|
||||
|
||||
def pkg_rules(self, pkg, tests):
|
||||
pkg = self.pkg_class(pkg)
|
||||
@ -1811,9 +1788,8 @@ def provider_defaults(self):
|
||||
|
||||
def provider_requirements(self):
|
||||
self.gen.h2("Requirements on virtual providers")
|
||||
parser = RequirementParser(spack.config.CONFIG)
|
||||
for virtual_str in sorted(self.possible_virtuals):
|
||||
rules = parser.rules_from_virtual(virtual_str)
|
||||
rules = self.requirement_parser.rules_from_virtual(virtual_str)
|
||||
if rules:
|
||||
self.emit_facts_from_requirement_rules(rules)
|
||||
self.trigger_rules()
|
||||
@ -3088,202 +3064,6 @@ def value(self) -> str:
|
||||
return "".join(self.asp_problem)
|
||||
|
||||
|
||||
def parse_spec_from_yaml_string(string: str) -> "spack.spec.Spec":
|
||||
"""Parse a spec from YAML and add file/line info to errors, if it's available.
|
||||
|
||||
Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and
|
||||
add file/line information for debugging using file/line annotations from the string.
|
||||
|
||||
Arguments:
|
||||
string: a string representing a ``Spec`` from config YAML.
|
||||
|
||||
"""
|
||||
try:
|
||||
return spack.spec.Spec(string)
|
||||
except SpecSyntaxError as e:
|
||||
mark = get_mark_from_yaml_data(string)
|
||||
if mark:
|
||||
msg = f"{mark.name}:{mark.line + 1}: {str(e)}"
|
||||
raise SpecSyntaxError(msg) from e
|
||||
raise e
|
||||
|
||||
|
||||
class RequirementParser:
|
||||
"""Parses requirements from package.py files and configuration, and returns rules."""
|
||||
|
||||
def __init__(self, configuration):
|
||||
self.config = configuration
|
||||
|
||||
def rules(self, pkg: "spack.package_base.PackageBase") -> List[RequirementRule]:
|
||||
result = []
|
||||
result.extend(self.rules_from_package_py(pkg))
|
||||
result.extend(self.rules_from_require(pkg))
|
||||
result.extend(self.rules_from_prefer(pkg))
|
||||
result.extend(self.rules_from_conflict(pkg))
|
||||
return result
|
||||
|
||||
def rules_from_package_py(self, pkg) -> List[RequirementRule]:
|
||||
rules = []
|
||||
for when_spec, requirement_list in pkg.requirements.items():
|
||||
for requirements, policy, message in requirement_list:
|
||||
rules.append(
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy=policy,
|
||||
requirements=requirements,
|
||||
kind=RequirementKind.PACKAGE,
|
||||
condition=when_spec,
|
||||
message=message,
|
||||
)
|
||||
)
|
||||
return rules
|
||||
|
||||
def rules_from_virtual(self, virtual_str: str) -> List[RequirementRule]:
|
||||
requirements = self.config.get("packages", {}).get(virtual_str, {}).get("require", [])
|
||||
return self._rules_from_requirements(
|
||||
virtual_str, requirements, kind=RequirementKind.VIRTUAL
|
||||
)
|
||||
|
||||
def rules_from_require(self, pkg: "spack.package_base.PackageBase") -> List[RequirementRule]:
|
||||
kind, requirements = self._raw_yaml_data(pkg, section="require")
|
||||
return self._rules_from_requirements(pkg.name, requirements, kind=kind)
|
||||
|
||||
def rules_from_prefer(self, pkg: "spack.package_base.PackageBase") -> List[RequirementRule]:
|
||||
result = []
|
||||
kind, preferences = self._raw_yaml_data(pkg, section="prefer")
|
||||
for item in preferences:
|
||||
spec, condition, message = self._parse_prefer_conflict_item(item)
|
||||
result.append(
|
||||
# A strong preference is defined as:
|
||||
#
|
||||
# require:
|
||||
# - any_of: [spec_str, "@:"]
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy="any_of",
|
||||
requirements=[spec, spack.spec.Spec("@:")],
|
||||
kind=kind,
|
||||
message=message,
|
||||
condition=condition,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def rules_from_conflict(self, pkg: "spack.package_base.PackageBase") -> List[RequirementRule]:
|
||||
result = []
|
||||
kind, conflicts = self._raw_yaml_data(pkg, section="conflict")
|
||||
for item in conflicts:
|
||||
spec, condition, message = self._parse_prefer_conflict_item(item)
|
||||
result.append(
|
||||
# A conflict is defined as:
|
||||
#
|
||||
# require:
|
||||
# - one_of: [spec_str, "@:"]
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy="one_of",
|
||||
requirements=[spec, spack.spec.Spec("@:")],
|
||||
kind=kind,
|
||||
message=message,
|
||||
condition=condition,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def _parse_prefer_conflict_item(self, item):
|
||||
# The item is either a string or an object with at least a "spec" attribute
|
||||
if isinstance(item, str):
|
||||
spec = parse_spec_from_yaml_string(item)
|
||||
condition = spack.spec.Spec()
|
||||
message = None
|
||||
else:
|
||||
spec = parse_spec_from_yaml_string(item["spec"])
|
||||
condition = spack.spec.Spec(item.get("when"))
|
||||
message = item.get("message")
|
||||
return spec, condition, message
|
||||
|
||||
def _raw_yaml_data(self, pkg: "spack.package_base.PackageBase", *, section: str):
|
||||
config = self.config.get("packages")
|
||||
data = config.get(pkg.name, {}).get(section, [])
|
||||
kind = RequirementKind.PACKAGE
|
||||
if not data:
|
||||
data = config.get("all", {}).get(section, [])
|
||||
kind = RequirementKind.DEFAULT
|
||||
return kind, data
|
||||
|
||||
def _rules_from_requirements(
|
||||
self, pkg_name: str, requirements, *, kind: RequirementKind
|
||||
) -> List[RequirementRule]:
|
||||
"""Manipulate requirements from packages.yaml, and return a list of tuples
|
||||
with a uniform structure (name, policy, requirements).
|
||||
"""
|
||||
if isinstance(requirements, str):
|
||||
requirements = [requirements]
|
||||
|
||||
rules = []
|
||||
for requirement in requirements:
|
||||
# A string is equivalent to a one_of group with a single element
|
||||
if isinstance(requirement, str):
|
||||
requirement = {"one_of": [requirement]}
|
||||
|
||||
for policy in ("spec", "one_of", "any_of"):
|
||||
if policy not in requirement:
|
||||
continue
|
||||
|
||||
constraints = requirement[policy]
|
||||
# "spec" is for specifying a single spec
|
||||
if policy == "spec":
|
||||
constraints = [constraints]
|
||||
policy = "one_of"
|
||||
|
||||
# validate specs from YAML first, and fail with line numbers if parsing fails.
|
||||
constraints = [
|
||||
parse_spec_from_yaml_string(constraint) for constraint in constraints
|
||||
]
|
||||
when_str = requirement.get("when")
|
||||
when = parse_spec_from_yaml_string(when_str) if when_str else spack.spec.Spec()
|
||||
|
||||
constraints = [
|
||||
x
|
||||
for x in constraints
|
||||
if not self.reject_requirement_constraint(pkg_name, constraint=x, kind=kind)
|
||||
]
|
||||
if not constraints:
|
||||
continue
|
||||
|
||||
rules.append(
|
||||
RequirementRule(
|
||||
pkg_name=pkg_name,
|
||||
policy=policy,
|
||||
requirements=constraints,
|
||||
kind=kind,
|
||||
message=requirement.get("message"),
|
||||
condition=when,
|
||||
)
|
||||
)
|
||||
return rules
|
||||
|
||||
def reject_requirement_constraint(
|
||||
self, pkg_name: str, *, constraint: spack.spec.Spec, kind: RequirementKind
|
||||
) -> bool:
|
||||
"""Returns True if a requirement constraint should be rejected"""
|
||||
if kind == RequirementKind.DEFAULT:
|
||||
# Requirements under all: are applied only if they are satisfiable considering only
|
||||
# package rules, so e.g. variants must exist etc. Otherwise, they are rejected.
|
||||
try:
|
||||
s = spack.spec.Spec(pkg_name)
|
||||
s.constrain(constraint)
|
||||
s.validate_or_raise()
|
||||
except spack.error.SpackError as e:
|
||||
tty.debug(
|
||||
f"[SETUP] Rejecting the default '{constraint}' requirement "
|
||||
f"on '{pkg_name}': {str(e)}",
|
||||
level=2,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class CompilerParser:
|
||||
"""Parses configuration files, and builds a list of possible compilers for the solve."""
|
||||
|
||||
|
||||
@ -1003,6 +1003,8 @@ variant_default_not_used(node(ID, Package), Variant, Value)
|
||||
node_has_variant(node(ID, Package), Variant, _),
|
||||
not attr("variant_value", node(ID, Package), Variant, Value),
|
||||
not propagate(node(ID, Package), variant_value(Variant, _, _)),
|
||||
% variant set explicitly don't count for this metric
|
||||
not attr("variant_set", node(ID, Package), Variant, _),
|
||||
attr("node", node(ID, Package)).
|
||||
|
||||
% The variant is set in an external spec
|
||||
|
||||
232
lib/spack/spack/solver/requirements.py
Normal file
232
lib/spack/spack/solver/requirements.py
Normal file
@ -0,0 +1,232 @@
|
||||
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
import enum
|
||||
from typing import List, NamedTuple, Optional, Sequence
|
||||
|
||||
from llnl.util import tty
|
||||
|
||||
import spack.config
|
||||
import spack.error
|
||||
import spack.package_base
|
||||
import spack.spec
|
||||
from spack.config import get_mark_from_yaml_data
|
||||
|
||||
|
||||
class RequirementKind(enum.Enum):
|
||||
"""Purpose / provenance of a requirement"""
|
||||
|
||||
#: Default requirement expressed under the 'all' attribute of packages.yaml
|
||||
DEFAULT = enum.auto()
|
||||
#: Requirement expressed on a virtual package
|
||||
VIRTUAL = enum.auto()
|
||||
#: Requirement expressed on a specific package
|
||||
PACKAGE = enum.auto()
|
||||
|
||||
|
||||
class RequirementRule(NamedTuple):
|
||||
"""Data class to collect information on a requirement"""
|
||||
|
||||
pkg_name: str
|
||||
policy: str
|
||||
requirements: Sequence[spack.spec.Spec]
|
||||
condition: spack.spec.Spec
|
||||
kind: RequirementKind
|
||||
message: Optional[str]
|
||||
|
||||
|
||||
class RequirementParser:
|
||||
"""Parses requirements from package.py files and configuration, and returns rules."""
|
||||
|
||||
def __init__(self, configuration: spack.config.Configuration):
|
||||
self.config = configuration
|
||||
|
||||
def rules(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
|
||||
result = []
|
||||
result.extend(self.rules_from_package_py(pkg))
|
||||
result.extend(self.rules_from_require(pkg))
|
||||
result.extend(self.rules_from_prefer(pkg))
|
||||
result.extend(self.rules_from_conflict(pkg))
|
||||
return result
|
||||
|
||||
def rules_from_package_py(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
|
||||
rules = []
|
||||
for when_spec, requirement_list in pkg.requirements.items():
|
||||
for requirements, policy, message in requirement_list:
|
||||
rules.append(
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy=policy,
|
||||
requirements=requirements,
|
||||
kind=RequirementKind.PACKAGE,
|
||||
condition=when_spec,
|
||||
message=message,
|
||||
)
|
||||
)
|
||||
return rules
|
||||
|
||||
def rules_from_virtual(self, virtual_str: str) -> List[RequirementRule]:
|
||||
requirements = self.config.get("packages", {}).get(virtual_str, {}).get("require", [])
|
||||
return self._rules_from_requirements(
|
||||
virtual_str, requirements, kind=RequirementKind.VIRTUAL
|
||||
)
|
||||
|
||||
def rules_from_require(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
|
||||
kind, requirements = self._raw_yaml_data(pkg, section="require")
|
||||
return self._rules_from_requirements(pkg.name, requirements, kind=kind)
|
||||
|
||||
def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
|
||||
result = []
|
||||
kind, preferences = self._raw_yaml_data(pkg, section="prefer")
|
||||
for item in preferences:
|
||||
spec, condition, message = self._parse_prefer_conflict_item(item)
|
||||
result.append(
|
||||
# A strong preference is defined as:
|
||||
#
|
||||
# require:
|
||||
# - any_of: [spec_str, "@:"]
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy="any_of",
|
||||
requirements=[spec, spack.spec.Spec("@:")],
|
||||
kind=kind,
|
||||
message=message,
|
||||
condition=condition,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def rules_from_conflict(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
|
||||
result = []
|
||||
kind, conflicts = self._raw_yaml_data(pkg, section="conflict")
|
||||
for item in conflicts:
|
||||
spec, condition, message = self._parse_prefer_conflict_item(item)
|
||||
result.append(
|
||||
# A conflict is defined as:
|
||||
#
|
||||
# require:
|
||||
# - one_of: [spec_str, "@:"]
|
||||
RequirementRule(
|
||||
pkg_name=pkg.name,
|
||||
policy="one_of",
|
||||
requirements=[spec, spack.spec.Spec("@:")],
|
||||
kind=kind,
|
||||
message=message,
|
||||
condition=condition,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def _parse_prefer_conflict_item(self, item):
|
||||
# The item is either a string or an object with at least a "spec" attribute
|
||||
if isinstance(item, str):
|
||||
spec = parse_spec_from_yaml_string(item)
|
||||
condition = spack.spec.Spec()
|
||||
message = None
|
||||
else:
|
||||
spec = parse_spec_from_yaml_string(item["spec"])
|
||||
condition = spack.spec.Spec(item.get("when"))
|
||||
message = item.get("message")
|
||||
return spec, condition, message
|
||||
|
||||
def _raw_yaml_data(self, pkg: spack.package_base.PackageBase, *, section: str):
|
||||
config = self.config.get("packages")
|
||||
data = config.get(pkg.name, {}).get(section, [])
|
||||
kind = RequirementKind.PACKAGE
|
||||
if not data:
|
||||
data = config.get("all", {}).get(section, [])
|
||||
kind = RequirementKind.DEFAULT
|
||||
return kind, data
|
||||
|
||||
def _rules_from_requirements(
|
||||
self, pkg_name: str, requirements, *, kind: RequirementKind
|
||||
) -> List[RequirementRule]:
|
||||
"""Manipulate requirements from packages.yaml, and return a list of tuples
|
||||
with a uniform structure (name, policy, requirements).
|
||||
"""
|
||||
if isinstance(requirements, str):
|
||||
requirements = [requirements]
|
||||
|
||||
rules = []
|
||||
for requirement in requirements:
|
||||
# A string is equivalent to a one_of group with a single element
|
||||
if isinstance(requirement, str):
|
||||
requirement = {"one_of": [requirement]}
|
||||
|
||||
for policy in ("spec", "one_of", "any_of"):
|
||||
if policy not in requirement:
|
||||
continue
|
||||
|
||||
constraints = requirement[policy]
|
||||
# "spec" is for specifying a single spec
|
||||
if policy == "spec":
|
||||
constraints = [constraints]
|
||||
policy = "one_of"
|
||||
|
||||
# validate specs from YAML first, and fail with line numbers if parsing fails.
|
||||
constraints = [
|
||||
parse_spec_from_yaml_string(constraint) for constraint in constraints
|
||||
]
|
||||
when_str = requirement.get("when")
|
||||
when = parse_spec_from_yaml_string(when_str) if when_str else spack.spec.Spec()
|
||||
|
||||
constraints = [
|
||||
x
|
||||
for x in constraints
|
||||
if not self.reject_requirement_constraint(pkg_name, constraint=x, kind=kind)
|
||||
]
|
||||
if not constraints:
|
||||
continue
|
||||
|
||||
rules.append(
|
||||
RequirementRule(
|
||||
pkg_name=pkg_name,
|
||||
policy=policy,
|
||||
requirements=constraints,
|
||||
kind=kind,
|
||||
message=requirement.get("message"),
|
||||
condition=when,
|
||||
)
|
||||
)
|
||||
return rules
|
||||
|
||||
def reject_requirement_constraint(
|
||||
self, pkg_name: str, *, constraint: spack.spec.Spec, kind: RequirementKind
|
||||
) -> bool:
|
||||
"""Returns True if a requirement constraint should be rejected"""
|
||||
if kind == RequirementKind.DEFAULT:
|
||||
# Requirements under all: are applied only if they are satisfiable considering only
|
||||
# package rules, so e.g. variants must exist etc. Otherwise, they are rejected.
|
||||
try:
|
||||
s = spack.spec.Spec(pkg_name)
|
||||
s.constrain(constraint)
|
||||
s.validate_or_raise()
|
||||
except spack.error.SpackError as e:
|
||||
tty.debug(
|
||||
f"[SETUP] Rejecting the default '{constraint}' requirement "
|
||||
f"on '{pkg_name}': {str(e)}",
|
||||
level=2,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def parse_spec_from_yaml_string(string: str) -> spack.spec.Spec:
|
||||
"""Parse a spec from YAML and add file/line info to errors, if it's available.
|
||||
|
||||
Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and
|
||||
add file/line information for debugging using file/line annotations from the string.
|
||||
|
||||
Arguments:
|
||||
string: a string representing a ``Spec`` from config YAML.
|
||||
|
||||
"""
|
||||
try:
|
||||
return spack.spec.Spec(string)
|
||||
except spack.error.SpecSyntaxError as e:
|
||||
mark = get_mark_from_yaml_data(string)
|
||||
if mark:
|
||||
msg = f"{mark.name}:{mark.line + 1}: {str(e)}"
|
||||
raise spack.error.SpecSyntaxError(msg) from e
|
||||
raise e
|
||||
Loading…
Reference in New Issue
Block a user