From 673b973c4165a970bb89a9ed9fd19f232ba4d3ac Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 9 Mar 2023 18:40:32 +0100 Subject: [PATCH] compilers: rework compiler encoding (add a compiler ID) --- lib/spack/spack/solver/asp.py | 118 ++++++++++++------------ lib/spack/spack/solver/concretize.lp | 130 ++++++++++++++++----------- 2 files changed, 138 insertions(+), 110 deletions(-) diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 8bee9d8557c..902ea1a303c 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -818,6 +818,9 @@ def __init__(self, tests=False): self.compiler_version_constraints = set() self.post_facts = [] + # (ID, CompilerSpec) -> dictionary of attributes + self.compiler_info = collections.defaultdict(dict) + # hashes we've already added facts for self.seen_hashes = set() self.reusable_and_possible = {} @@ -945,36 +948,59 @@ def conflict_rules(self, pkg): def compiler_facts(self): """Facts about available compilers.""" + # Build a dictionary mapping a compiler spec to supported OS and targets + for entry in spack.compilers.all_compilers_config(): + compiler_entry = entry["compiler"] + key = spack.spec.CompilerSpec(compiler_entry["spec"]) + + # FIXME: same spec with multiple OS and targets? + if key in self.compiler_info: + warnings.warn( + f"compiler {key} has duplicate entries, only the first one will be considered" + ) + continue + + self.compiler_info[key]["os"] = compiler_entry["operating_system"] + self.compiler_info[key]["target"] = compiler_entry.get("target", None) + self.compiler_info[key]["compiler_obj"] = spack.compilers.compiler_from_dict( + compiler_entry + ) + self.gen.h2("Available compilers") - for compiler_id, compiler in enumerate(self.possible_compilers): - self.gen.fact(fn.compiler_version(compiler.name, compiler.version)) + indexed_possible_compilers = list(enumerate(self.possible_compilers)) + for compiler_id, compiler in indexed_possible_compilers: + self.gen.fact(fn.compiler_id(compiler_id)) + self.gen.fact(fn.compiler_name(compiler_id, compiler.name)) + self.gen.fact(fn.compiler_version(compiler_id, compiler.version)) + + compiler_entry = self.compiler_info[compiler] + # Use "get" to allow concretizing even when copiler existence checks are disabled + operating_system = compiler_entry.get("os") + if operating_system: + self.gen.fact(fn.compiler_os(compiler_id, operating_system)) + + target = compiler_entry.get("target") + if target is not None: + self.gen.fact(fn.compiler_target(compiler_id, target)) + + compiler_obj = compiler_entry.get("compiler_obj") + if compiler_obj: + for flag_type, flags in compiler_obj.flags.items(): + for flag in flags: + self.gen.fact(fn.compiler_flag(compiler_id, flag_type, flag)) + self.gen.newline() # Set compiler defaults, given a list of possible compilers - self.gen.h2("Default compiler preferences") + self.gen.h2("Default compiler preferences (CompilerID, Weight)") ppk = spack.package_prefs.PackagePrefs("all", "compiler", all=False) - matches = sorted(self.possible_compilers, key=ppk) + matches = sorted(indexed_possible_compilers, key=lambda x: ppk(x[1])) - for i, cspec in enumerate(matches): - f = fn.default_compiler_preference(cspec.name, cspec.version, i) + for weight, (compiler_id, cspec) in enumerate(matches): + f = fn.default_compiler_preference(compiler_id, weight) self.gen.fact(f) - # Enumerate target families. This may be redundant, but compilers with - # custom versions will be able to concretize properly. - for entry in spack.compilers.all_compilers_config(): - compiler_entry = entry["compiler"] - cspec = spack.spec.CompilerSpec(compiler_entry["spec"]) - operating_system = compiler_entry["operating_system"] - self.gen.fact(fn.compiler_supports_os(cspec.name, cspec.version, operating_system)) - - if not compiler_entry.get("target", None): - continue - - self.gen.fact( - fn.compiler_supports_target(cspec.name, cspec.version, compiler_entry["target"]) - ) - def package_compiler_defaults(self, pkg): """Facts about packages' compiler prefs.""" @@ -1377,28 +1403,6 @@ def target_preferences(self, pkg_name): fn.target_weight(pkg_name, str(preferred.architecture.target), i + offset) ) - def flag_defaults(self): - self.gen.h2("Compiler flag defaults") - - # types of flags that can be on specs - for flag in spack.spec.FlagMap.valid_compiler_flags(): - self.gen.fact(fn.flag_type(flag)) - self.gen.newline() - - # flags from compilers.yaml - compilers = all_compilers_in_config() - seen = set() - for compiler in compilers: - # if there are multiple with the same spec, only use the first - if compiler.spec in seen: - continue - seen.add(compiler.spec) - for name, flags in compiler.flags.items(): - for flag in flags: - self.gen.fact( - fn.compiler_version_flag(compiler.name, compiler.version, name, flag) - ) - def spec_clauses(self, *args, **kwargs): """Wrap a call to `_spec_clauses()` into a try/except block that raises a comprehensible error message in case of failure. @@ -1755,8 +1759,6 @@ def target_defaults(self, specs): if granularity == "generic": candidate_targets = [t for t in candidate_targets if t.vendor == "generic"] - compilers = self.possible_compilers - # Add targets explicitly requested from specs for spec in specs: if not spec.architecture or not spec.architecture.target: @@ -1774,7 +1776,7 @@ def target_defaults(self, specs): candidate_targets.append(ancestor) best_targets = set([uarch.family.name]) - for compiler in sorted(compilers): + for compiler_id, compiler in enumerate(self.possible_compilers): supported = self._supported_targets(compiler.name, compiler.version, candidate_targets) # If we can't find supported targets it may be due to custom @@ -1793,13 +1795,10 @@ def target_defaults(self, specs): for target in supported: best_targets.add(target.name) - self.gen.fact( - fn.compiler_supports_target(compiler.name, compiler.version, target.name) - ) + self.gen.fact(fn.compiler_supports_target(compiler_id, target.name)) - self.gen.fact( - fn.compiler_supports_target(compiler.name, compiler.version, uarch.family.name) - ) + self.gen.fact(fn.compiler_supports_target(compiler_id, uarch.family.name)) + self.gen.newline() i = 0 # TODO compute per-target offset? for target in candidate_targets: @@ -1859,7 +1858,6 @@ def generate_possible_compilers(self, specs): # is already built else: cspecs.add(s.compiler) - # FIXME (COMPILERS) self.gen.fact(fn.allow_compiler(s.compiler.name, s.compiler.version)) return list( @@ -1921,14 +1919,12 @@ def versions_for(v): self.possible_versions[pkg_name].add(version) def define_compiler_version_constraints(self): - compiler_list = spack.compilers.all_compiler_specs() - compiler_list = list(sorted(set(compiler_list))) for constraint in sorted(self.compiler_version_constraints): - for compiler in compiler_list: + for compiler_id, compiler in enumerate(self.possible_compilers): if compiler.satisfies(constraint): self.gen.fact( fn.compiler_version_satisfies( - constraint.name, constraint.versions, compiler.version + constraint.name, constraint.versions, compiler_id ) ) self.gen.newline() @@ -2089,6 +2085,11 @@ def setup(self, driver, specs, reuse=None): for reusable_spec in reuse: self._facts_from_concrete_spec(reusable_spec, possible) + self.gen.h1("Possible flags on nodes") + for flag in spack.spec.FlagMap.valid_compiler_flags(): + self.gen.fact(fn.flag_type(flag)) + self.gen.newline() + self.gen.h1("General Constraints") self.compiler_facts() @@ -2101,7 +2102,6 @@ def setup(self, driver, specs, reuse=None): self.provider_defaults() self.provider_requirements() self.external_packages() - self.flag_defaults() self.gen.h1("Package Constraints") for pkg in sorted(self.pkgs): @@ -2282,7 +2282,7 @@ def reorder_flags(self): flags will appear last on the compile line, in the order they were specified. - The solver determines wihch flags are on nodes; this routine + The solver determines which flags are on nodes; this routine imposes order afterwards. """ # reverse compilers so we get highest priority compilers that share a spec diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 2ec997fe8b2..bbcf20e7e0a 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -821,9 +821,10 @@ node_target_compatible(Package, Target) % can't use targets on node if the compiler for the node doesn't support them error(2, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Target, Compiler, Version) :- attr("node_target", Package, Target), - not compiler_supports_target(Compiler, Version, Target), - attr("node_compiler", Package, Compiler), - attr("node_compiler_version", Package, Compiler, Version), + node_compiler(Package, CompilerID), + not compiler_supports_target(CompilerID, Target), + compiler_name(CompilerID, Compiler), + compiler_version(CompilerID, Version), build(Package). % if a target is set explicitly, respect it @@ -857,32 +858,44 @@ error(2, "'{0} target={1}' is not compatible with this machine", Package, Target %----------------------------------------------------------------------------- % Compiler semantics %----------------------------------------------------------------------------- -compiler(Compiler) :- compiler_version(Compiler, _). - -% There must be only one compiler set per built node. The compiler -% is chosen among available versions. -{ attr("node_compiler_version", Package, Compiler, Version) : compiler_version(Compiler, Version) } :- +% There must be only one compiler set per built node. +{ node_compiler(Package, CompilerID) : compiler_id(CompilerID) } :- attr("node", Package), build(Package). +% Infer the compiler that matches a reused node +node_compiler(Package, CompilerID) + :- attr("node_compiler_version", Package, CompilerName, CompilerVersion), + attr("node", Package), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, CompilerVersion), + not build(Package). + +% Expand the internal attribute into "attr("node_compiler_version") +attr("node_compiler_version", Package, CompilerName, CompilerVersion) + :- node_compiler(Package, CompilerID), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, CompilerVersion), + build(Package). + +attr("node_compiler", Package, CompilerName) + :- attr("node_compiler_version", Package, CompilerName, CompilerVersion). + error(2, "No valid compiler version found for '{0}'", Package) :- attr("node", Package), - C = #count{ Version : attr("node_compiler_version", Package, _, Version)}, - C < 1. -error(2, "'{0}' compiler constraints '%{1}@{2}' and '%{3}@{4}' are incompatible", Package, Compiler1, Version1, Compiler2, Version2) - :- attr("node", Package), - attr("node_compiler_version", Package, Compiler1, Version1), - attr("node_compiler_version", Package, Compiler2, Version2), - (Compiler1, Version1) < (Compiler2, Version2). % see[1] + not attr("node_compiler_version", Package, _, _). -% Sometimes we just need to know the compiler and not the version -attr("node_compiler", Package, Compiler) :- attr("node_compiler_version", Package, Compiler, _). +error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2) + :- attr("node", Package), + attr("node_compiler_version", Package, C1, V1), + attr("node_compiler_version", Package, C2, V2), + (C1, V1) < (C2, V2). % see[1] % We can't have a compiler be enforced and select the version from another compiler error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2) :- attr("node_compiler_version", Package, C1, V1), attr("node_compiler_version", Package, C2, V2), - (C1, V1) != (C2, V2). + (C1, V1) < (C2, V2). error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Compiler1, Compiler2, Version) :- attr("node_compiler", Package, Compiler1), @@ -893,37 +906,41 @@ error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Co error(1, "No valid compiler for {0} satisfies '%{1}'", Package, Compiler) :- attr("node", Package), attr("node_compiler_version_satisfies", Package, Compiler, ":"), - C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, ":", Version) }, - C < 1. + not compiler_version_satisfies(Compiler, ":", _). % If the compiler of a node must satisfy a constraint, then its version % must be chosen among the ones that satisfy said constraint error(2, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint) :- attr("node", Package), attr("node_compiler_version_satisfies", Package, Compiler, Constraint), - C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, Constraint, Version) }, - C < 1. + not compiler_version_satisfies(Compiler, Constraint, _). % If the node is associated with a compiler and the compiler satisfy a constraint, then % the compiler associated with the node satisfy the same constraint attr("node_compiler_version_satisfies", Package, Compiler, Constraint) - :- attr("node_compiler_version", Package, Compiler, Version), - compiler_version_satisfies(Compiler, Constraint, Version). + :- node_compiler(Package, CompilerID), + compiler_name(CompilerID, Compiler), + compiler_version_satisfies(Compiler, Constraint, CompilerID). #defined compiler_version_satisfies/3. % If the compiler version was set from the command line, % respect it verbatim -attr("node_compiler_version", Package, Compiler, Version) :- - attr("node_compiler_version_set", Package, Compiler, Version). +:- attr("node_compiler_version_set", Package, Compiler, Version), + not attr("node_compiler_version", Package, Compiler, Version). + +:- attr("node_compiler_set", Package, Compiler), + not attr("node_compiler_version", Package, Compiler, _). % Cannot select a compiler if it is not supported on the OS % Compilers that are explicitly marked as allowed % are excluded from this check error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler, Version, OS) - :- attr("node_compiler_version", Package, Compiler, Version), - attr("node_os", Package, OS), - not compiler_supports_os(Compiler, Version, OS), + :- attr("node_os", Package, OS), + node_compiler(Package, CompilerID), + compiler_name(CompilerID, Compiler), + compiler_version(CompilerID, Version), + not compiler_os(CompilerID, OS), not allow_compiler(Compiler, Version), build(Package). @@ -931,8 +948,8 @@ error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler % same compiler there's a mismatch. compiler_match(Package, Dependency) :- depends_on(Package, Dependency), - attr("node_compiler_version", Package, Compiler, Version), - attr("node_compiler_version", Dependency, Compiler, Version). + node_compiler(Package, CompilerID), + node_compiler(Dependency, CompilerID). compiler_mismatch(Package, Dependency) :- depends_on(Package, Dependency), @@ -944,25 +961,32 @@ compiler_mismatch_required(Package, Dependency) attr("node_compiler_set", Dependency, _), not compiler_match(Package, Dependency). -#defined compiler_supports_os/3. +#defined compiler_os/3. #defined allow_compiler/2. % compilers weighted by preference according to packages.yaml compiler_weight(Package, Weight) - :- attr("node_compiler_version", Package, Compiler, V), + :- node_compiler(Package, CompilerID), + compiler_name(CompilerID, Compiler), + compiler_version(CompilerID, V), node_compiler_preference(Package, Compiler, V, Weight). compiler_weight(Package, Weight) - :- attr("node_compiler_version", Package, Compiler, V), + :- node_compiler(Package, CompilerID), + compiler_name(CompilerID, Compiler), + compiler_version(CompilerID, V), not node_compiler_preference(Package, Compiler, V, _), - default_compiler_preference(Compiler, V, Weight). + default_compiler_preference(CompilerID, Weight). compiler_weight(Package, 100) - :- attr("node_compiler_version", Package, Compiler, Version), - not node_compiler_preference(Package, Compiler, Version, _), - not default_compiler_preference(Compiler, Version, _). + :- node_compiler(Package, CompilerID), + compiler_name(CompilerID, Compiler), + compiler_version(CompilerID, V), + not node_compiler_preference(Package, Compiler, V, _), + not default_compiler_preference(CompilerID, _). % For the time being, be strict and reuse only if the compiler match one we have on the system error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missing_compilers:true if intended.", Package, Compiler, Version) - :- attr("node_compiler_version", Package, Compiler, Version), not compiler_version(Compiler, Version). + :- attr("node_compiler_version", Package, Compiler, Version), + not node_compiler(Package, _). #defined node_compiler_preference/4. #defined default_compiler_preference/3. @@ -974,10 +998,11 @@ error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missin % propagate flags when compiler match can_inherit_flags(Package, Dependency, FlagType) :- depends_on(Package, Dependency), - attr("node_compiler", Package, Compiler), - attr("node_compiler", Dependency, Compiler), + node_compiler(Package, CompilerID), + node_compiler(Dependency, CompilerID), not attr("node_flag_set", Dependency, FlagType, _), - compiler(Compiler), flag_type(FlagType). + compiler_id(CompilerID), + flag_type(FlagType). node_flag_inherited(Dependency, FlagType, Flag) :- attr("node_flag_set", Package, FlagType, Flag), can_inherit_flags(Package, Dependency, FlagType), @@ -1004,19 +1029,21 @@ attr("node_flag_source", Dependency, FlagType, Q) % compiler flags from compilers.yaml are put on nodes if compiler matches attr("node_flag", Package, FlagType, Flag) - :- compiler_version_flag(Compiler, Version, FlagType, Flag), - attr("node_compiler_version", Package, Compiler, Version), + :- compiler_flag(CompilerID, FlagType, Flag), + node_compiler(Package, CompilerID), flag_type(FlagType), - compiler(Compiler), - compiler_version(Compiler, Version). + compiler_id(CompilerID), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, Version). attr("node_flag_compiler_default", Package) :- not attr("node_flag_set", Package, FlagType, _), - compiler_version_flag(Compiler, Version, FlagType, Flag), - attr("node_compiler_version", Package, Compiler, Version), + compiler_flag(CompilerID, FlagType, Flag), + node_compiler(Package, CompilerID), flag_type(FlagType), - compiler(Compiler), - compiler_version(Compiler, Version). + compiler_id(CompilerID), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, Version). % if a flag is set to something or inherited, it's included attr("node_flag", Package, FlagType, Flag) :- attr("node_flag_set", Package, FlagType, Flag). @@ -1027,7 +1054,7 @@ attr("node_flag", Package, FlagType, Flag) attr("no_flags", Package, FlagType) :- not attr("node_flag", Package, FlagType, _), attr("node", Package), flag_type(FlagType). -#defined compiler_version_flag/4. +#defined compiler_flag/3. %----------------------------------------------------------------------------- @@ -1308,6 +1335,7 @@ opt_criterion(5, "non-preferred targets"). #heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] #heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] #heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true] +#heuristic attr("node_compiler_version", Package, Compiler, Version) : default_compiler_preference(ID, 0), compiler_name(ID, Compiler), compiler_version(ID, Version), attr("node", Package). [10, true] %----------- % Notes