zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compilers: rework compiler encoding (add a compiler ID)

Browse Source
This commit is contained in:
Massimiliano Culpo 2023-03-09 18:40:32 +01:00
parent b4db7de4c9
commit 673b973c41
No known key found for this signature in database
GPG Key ID: 3E52BB992233066C
2 changed files with 138 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
lib/spack/spack/solver/asp.py
View File

@ -818,6 +818,9 @@ def __init__(self, tests=False):
self.compiler_version_constraints = set()
self.post_facts = []
# (ID, CompilerSpec) -> dictionary of attributes
self.compiler_info = collections.defaultdict(dict)
# hashes we've already added facts for
self.seen_hashes = set()
self.reusable_and_possible = {}
@ -945,36 +948,59 @@ def conflict_rules(self, pkg):
def compiler_facts(self):
"""Facts about available compilers."""
# Build a dictionary mapping a compiler spec to supported OS and targets
for entry in spack.compilers.all_compilers_config():
compiler_entry = entry["compiler"]
key = spack.spec.CompilerSpec(compiler_entry["spec"])
# FIXME: same spec with multiple OS and targets?
if key in self.compiler_info:
warnings.warn(
f"compiler {key} has duplicate entries, only the first one will be considered"
)
continue
self.compiler_info[key]["os"] = compiler_entry["operating_system"]
self.compiler_info[key]["target"] = compiler_entry.get("target", None)
self.compiler_info[key]["compiler_obj"] = spack.compilers.compiler_from_dict(
compiler_entry
)
self.gen.h2("Available compilers")
for compiler_id, compiler in enumerate(self.possible_compilers):
self.gen.fact(fn.compiler_version(compiler.name, compiler.version))
indexed_possible_compilers = list(enumerate(self.possible_compilers))
for compiler_id, compiler in indexed_possible_compilers:
self.gen.fact(fn.compiler_id(compiler_id))
self.gen.fact(fn.compiler_name(compiler_id, compiler.name))
self.gen.fact(fn.compiler_version(compiler_id, compiler.version))
compiler_entry = self.compiler_info[compiler]
# Use "get" to allow concretizing even when copiler existence checks are disabled
operating_system = compiler_entry.get("os")
if operating_system:
self.gen.fact(fn.compiler_os(compiler_id, operating_system))
target = compiler_entry.get("target")
if target is not None:
self.gen.fact(fn.compiler_target(compiler_id, target))
compiler_obj = compiler_entry.get("compiler_obj")
if compiler_obj:
for flag_type, flags in compiler_obj.flags.items():
for flag in flags:
self.gen.fact(fn.compiler_flag(compiler_id, flag_type, flag))
self.gen.newline()
# Set compiler defaults, given a list of possible compilers
self.gen.h2("Default compiler preferences")
self.gen.h2("Default compiler preferences (CompilerID, Weight)")
ppk = spack.package_prefs.PackagePrefs("all", "compiler", all=False)
matches = sorted(self.possible_compilers, key=ppk)
matches = sorted(indexed_possible_compilers, key=lambda x: ppk(x[1]))
for i, cspec in enumerate(matches):
f = fn.default_compiler_preference(cspec.name, cspec.version, i)
for weight, (compiler_id, cspec) in enumerate(matches):
f = fn.default_compiler_preference(compiler_id, weight)
self.gen.fact(f)
# Enumerate target families. This may be redundant, but compilers with
# custom versions will be able to concretize properly.
for entry in spack.compilers.all_compilers_config():
compiler_entry = entry["compiler"]
cspec = spack.spec.CompilerSpec(compiler_entry["spec"])
operating_system = compiler_entry["operating_system"]
self.gen.fact(fn.compiler_supports_os(cspec.name, cspec.version, operating_system))
if not compiler_entry.get("target", None):
continue
self.gen.fact(
fn.compiler_supports_target(cspec.name, cspec.version, compiler_entry["target"])
)
def package_compiler_defaults(self, pkg):
"""Facts about packages' compiler prefs."""
@ -1377,28 +1403,6 @@ def target_preferences(self, pkg_name):
fn.target_weight(pkg_name, str(preferred.architecture.target), i + offset)
)
def flag_defaults(self):
self.gen.h2("Compiler flag defaults")
# types of flags that can be on specs
for flag in spack.spec.FlagMap.valid_compiler_flags():
self.gen.fact(fn.flag_type(flag))
self.gen.newline()
# flags from compilers.yaml
compilers = all_compilers_in_config()
seen = set()
for compiler in compilers:
# if there are multiple with the same spec, only use the first
if compiler.spec in seen:
continue
seen.add(compiler.spec)
for name, flags in compiler.flags.items():
for flag in flags:
self.gen.fact(
fn.compiler_version_flag(compiler.name, compiler.version, name, flag)
)
def spec_clauses(self, *args, **kwargs):
"""Wrap a call to `_spec_clauses()` into a try/except block that
raises a comprehensible error message in case of failure.
@ -1755,8 +1759,6 @@ def target_defaults(self, specs):
if granularity == "generic":
candidate_targets = [t for t in candidate_targets if t.vendor == "generic"]
compilers = self.possible_compilers
# Add targets explicitly requested from specs
for spec in specs:
if not spec.architecture or not spec.architecture.target:
@ -1774,7 +1776,7 @@ def target_defaults(self, specs):
candidate_targets.append(ancestor)
best_targets = set([uarch.family.name])
for compiler in sorted(compilers):
for compiler_id, compiler in enumerate(self.possible_compilers):
supported = self._supported_targets(compiler.name, compiler.version, candidate_targets)
# If we can't find supported targets it may be due to custom
@ -1793,13 +1795,10 @@ def target_defaults(self, specs):
for target in supported:
best_targets.add(target.name)
self.gen.fact(
fn.compiler_supports_target(compiler.name, compiler.version, target.name)
)
self.gen.fact(fn.compiler_supports_target(compiler_id, target.name))
self.gen.fact(
fn.compiler_supports_target(compiler.name, compiler.version, uarch.family.name)
)
self.gen.fact(fn.compiler_supports_target(compiler_id, uarch.family.name))
self.gen.newline()
i = 0 # TODO compute per-target offset?
for target in candidate_targets:
@ -1859,7 +1858,6 @@ def generate_possible_compilers(self, specs):
# is already built
else:
cspecs.add(s.compiler)
# FIXME (COMPILERS)
self.gen.fact(fn.allow_compiler(s.compiler.name, s.compiler.version))
return list(
@ -1921,14 +1919,12 @@ def versions_for(v):
self.possible_versions[pkg_name].add(version)
def define_compiler_version_constraints(self):
compiler_list = spack.compilers.all_compiler_specs()
compiler_list = list(sorted(set(compiler_list)))
for constraint in sorted(self.compiler_version_constraints):
for compiler in compiler_list:
for compiler_id, compiler in enumerate(self.possible_compilers):
if compiler.satisfies(constraint):
self.gen.fact(
fn.compiler_version_satisfies(
constraint.name, constraint.versions, compiler.version
constraint.name, constraint.versions, compiler_id
)
)
self.gen.newline()
@ -2089,6 +2085,11 @@ def setup(self, driver, specs, reuse=None):
for reusable_spec in reuse:
self._facts_from_concrete_spec(reusable_spec, possible)
self.gen.h1("Possible flags on nodes")
for flag in spack.spec.FlagMap.valid_compiler_flags():
self.gen.fact(fn.flag_type(flag))
self.gen.newline()
self.gen.h1("General Constraints")
self.compiler_facts()
@ -2101,7 +2102,6 @@ def setup(self, driver, specs, reuse=None):
self.provider_defaults()
self.provider_requirements()
self.external_packages()
self.flag_defaults()
self.gen.h1("Package Constraints")
for pkg in sorted(self.pkgs):
@ -2282,7 +2282,7 @@ def reorder_flags(self):
flags will appear last on the compile line, in the order they
were specified.
The solver determines wihch flags are on nodes; this routine
The solver determines which flags are on nodes; this routine
imposes order afterwards.
"""
# reverse compilers so we get highest priority compilers that share a spec

130
lib/spack/spack/solver/concretize.lp
View File

@ -821,9 +821,10 @@ node_target_compatible(Package, Target)
% can't use targets on node if the compiler for the node doesn't support them
error(2, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Target, Compiler, Version)
:- attr("node_target", Package, Target),
not compiler_supports_target(Compiler, Version, Target),
attr("node_compiler", Package, Compiler),
attr("node_compiler_version", Package, Compiler, Version),
node_compiler(Package, CompilerID),
not compiler_supports_target(CompilerID, Target),
compiler_name(CompilerID, Compiler),
compiler_version(CompilerID, Version),
build(Package).
% if a target is set explicitly, respect it
@ -857,32 +858,44 @@ error(2, "'{0} target={1}' is not compatible with this machine", Package, Target
%-----------------------------------------------------------------------------
% Compiler semantics
%-----------------------------------------------------------------------------
compiler(Compiler) :- compiler_version(Compiler, _).
% There must be only one compiler set per built node. The compiler
% is chosen among available versions.
{ attr("node_compiler_version", Package, Compiler, Version) : compiler_version(Compiler, Version) } :-
% There must be only one compiler set per built node.
{ node_compiler(Package, CompilerID) : compiler_id(CompilerID) } :-
attr("node", Package),
build(Package).
% Infer the compiler that matches a reused node
node_compiler(Package, CompilerID)
:- attr("node_compiler_version", Package, CompilerName, CompilerVersion),
attr("node", Package),
compiler_name(CompilerID, CompilerName),
compiler_version(CompilerID, CompilerVersion),
not build(Package).
% Expand the internal attribute into "attr("node_compiler_version")
attr("node_compiler_version", Package, CompilerName, CompilerVersion)
:- node_compiler(Package, CompilerID),
compiler_name(CompilerID, CompilerName),
compiler_version(CompilerID, CompilerVersion),
build(Package).
attr("node_compiler", Package, CompilerName)
:- attr("node_compiler_version", Package, CompilerName, CompilerVersion).
error(2, "No valid compiler version found for '{0}'", Package)
:- attr("node", Package),
C = #count{ Version : attr("node_compiler_version", Package, _, Version)},
C < 1.
error(2, "'{0}' compiler constraints '%{1}@{2}' and '%{3}@{4}' are incompatible", Package, Compiler1, Version1, Compiler2, Version2)
:- attr("node", Package),
attr("node_compiler_version", Package, Compiler1, Version1),
attr("node_compiler_version", Package, Compiler2, Version2),
(Compiler1, Version1) < (Compiler2, Version2). % see[1]
not attr("node_compiler_version", Package, _, _).
% Sometimes we just need to know the compiler and not the version
attr("node_compiler", Package, Compiler) :- attr("node_compiler_version", Package, Compiler, _).
error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2)
:- attr("node", Package),
attr("node_compiler_version", Package, C1, V1),
attr("node_compiler_version", Package, C2, V2),
(C1, V1) < (C2, V2). % see[1]
% We can't have a compiler be enforced and select the version from another compiler
error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2)
:- attr("node_compiler_version", Package, C1, V1),
attr("node_compiler_version", Package, C2, V2),
(C1, V1) != (C2, V2).
(C1, V1) < (C2, V2).
error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Compiler1, Compiler2, Version)
:- attr("node_compiler", Package, Compiler1),
@ -893,37 +906,41 @@ error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Co
error(1, "No valid compiler for {0} satisfies '%{1}'", Package, Compiler)
:- attr("node", Package),
attr("node_compiler_version_satisfies", Package, Compiler, ":"),
C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, ":", Version) },
C < 1.
not compiler_version_satisfies(Compiler, ":", _).
% If the compiler of a node must satisfy a constraint, then its version
% must be chosen among the ones that satisfy said constraint
error(2, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint)
:- attr("node", Package),
attr("node_compiler_version_satisfies", Package, Compiler, Constraint),
C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, Constraint, Version) },
C < 1.
not compiler_version_satisfies(Compiler, Constraint, _).
% If the node is associated with a compiler and the compiler satisfy a constraint, then
% the compiler associated with the node satisfy the same constraint
attr("node_compiler_version_satisfies", Package, Compiler, Constraint)
:- attr("node_compiler_version", Package, Compiler, Version),
compiler_version_satisfies(Compiler, Constraint, Version).
:- node_compiler(Package, CompilerID),
compiler_name(CompilerID, Compiler),
compiler_version_satisfies(Compiler, Constraint, CompilerID).
#defined compiler_version_satisfies/3.
% If the compiler version was set from the command line,
% respect it verbatim
attr("node_compiler_version", Package, Compiler, Version) :-
attr("node_compiler_version_set", Package, Compiler, Version).
:- attr("node_compiler_version_set", Package, Compiler, Version),
not attr("node_compiler_version", Package, Compiler, Version).
:- attr("node_compiler_set", Package, Compiler),
not attr("node_compiler_version", Package, Compiler, _).
% Cannot select a compiler if it is not supported on the OS
% Compilers that are explicitly marked as allowed
% are excluded from this check
error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler, Version, OS)
:- attr("node_compiler_version", Package, Compiler, Version),
attr("node_os", Package, OS),
not compiler_supports_os(Compiler, Version, OS),
:- attr("node_os", Package, OS),
node_compiler(Package, CompilerID),
compiler_name(CompilerID, Compiler),
compiler_version(CompilerID, Version),
not compiler_os(CompilerID, OS),
not allow_compiler(Compiler, Version),
build(Package).
@ -931,8 +948,8 @@ error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler
% same compiler there's a mismatch.
compiler_match(Package, Dependency)
:- depends_on(Package, Dependency),
attr("node_compiler_version", Package, Compiler, Version),
attr("node_compiler_version", Dependency, Compiler, Version).
node_compiler(Package, CompilerID),
node_compiler(Dependency, CompilerID).
compiler_mismatch(Package, Dependency)
:- depends_on(Package, Dependency),
@ -944,25 +961,32 @@ compiler_mismatch_required(Package, Dependency)
attr("node_compiler_set", Dependency, _),
not compiler_match(Package, Dependency).
#defined compiler_supports_os/3.
#defined compiler_os/3.
#defined allow_compiler/2.
% compilers weighted by preference according to packages.yaml
compiler_weight(Package, Weight)
:- attr("node_compiler_version", Package, Compiler, V),
:- node_compiler(Package, CompilerID),
compiler_name(CompilerID, Compiler),
compiler_version(CompilerID, V),
node_compiler_preference(Package, Compiler, V, Weight).
compiler_weight(Package, Weight)
:- attr("node_compiler_version", Package, Compiler, V),
:- node_compiler(Package, CompilerID),
compiler_name(CompilerID, Compiler),
compiler_version(CompilerID, V),
not node_compiler_preference(Package, Compiler, V, _),
default_compiler_preference(Compiler, V, Weight).
default_compiler_preference(CompilerID, Weight).
compiler_weight(Package, 100)
:- attr("node_compiler_version", Package, Compiler, Version),
not node_compiler_preference(Package, Compiler, Version, _),
not default_compiler_preference(Compiler, Version, _).
:- node_compiler(Package, CompilerID),
compiler_name(CompilerID, Compiler),
compiler_version(CompilerID, V),
not node_compiler_preference(Package, Compiler, V, _),
not default_compiler_preference(CompilerID, _).
% For the time being, be strict and reuse only if the compiler match one we have on the system
error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missing_compilers:true if intended.", Package, Compiler, Version)
:- attr("node_compiler_version", Package, Compiler, Version), not compiler_version(Compiler, Version).
:- attr("node_compiler_version", Package, Compiler, Version),
not node_compiler(Package, _).
#defined node_compiler_preference/4.
#defined default_compiler_preference/3.
@ -974,10 +998,11 @@ error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missin
% propagate flags when compiler match
can_inherit_flags(Package, Dependency, FlagType)
:- depends_on(Package, Dependency),
attr("node_compiler", Package, Compiler),
attr("node_compiler", Dependency, Compiler),
node_compiler(Package, CompilerID),
node_compiler(Dependency, CompilerID),
not attr("node_flag_set", Dependency, FlagType, _),
compiler(Compiler), flag_type(FlagType).
compiler_id(CompilerID),
flag_type(FlagType).
node_flag_inherited(Dependency, FlagType, Flag)
:- attr("node_flag_set", Package, FlagType, Flag), can_inherit_flags(Package, Dependency, FlagType),
@ -1004,19 +1029,21 @@ attr("node_flag_source", Dependency, FlagType, Q)
% compiler flags from compilers.yaml are put on nodes if compiler matches
attr("node_flag", Package, FlagType, Flag)
:- compiler_version_flag(Compiler, Version, FlagType, Flag),
attr("node_compiler_version", Package, Compiler, Version),
:- compiler_flag(CompilerID, FlagType, Flag),
node_compiler(Package, CompilerID),
flag_type(FlagType),
compiler(Compiler),
compiler_version(Compiler, Version).
compiler_id(CompilerID),
compiler_name(CompilerID, CompilerName),
compiler_version(CompilerID, Version).
attr("node_flag_compiler_default", Package)
:- not attr("node_flag_set", Package, FlagType, _),
compiler_version_flag(Compiler, Version, FlagType, Flag),
attr("node_compiler_version", Package, Compiler, Version),
compiler_flag(CompilerID, FlagType, Flag),
node_compiler(Package, CompilerID),
flag_type(FlagType),
compiler(Compiler),
compiler_version(Compiler, Version).
compiler_id(CompilerID),
compiler_name(CompilerID, CompilerName),
compiler_version(CompilerID, Version).
% if a flag is set to something or inherited, it's included
attr("node_flag", Package, FlagType, Flag) :- attr("node_flag_set", Package, FlagType, Flag).
@ -1027,7 +1054,7 @@ attr("node_flag", Package, FlagType, Flag)
attr("no_flags", Package, FlagType)
:- not attr("node_flag", Package, FlagType, _), attr("node", Package), flag_type(FlagType).
#defined compiler_version_flag/4.
#defined compiler_flag/3.
%-----------------------------------------------------------------------------
@ -1308,6 +1335,7 @@ opt_criterion(5, "non-preferred targets").
#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true]
#heuristic attr("node_compiler_version", Package, Compiler, Version) : default_compiler_preference(ID, 0), compiler_name(ID, Compiler), compiler_version(ID, Version), attr("node", Package). [10, true]
%-----------
% Notes