compilers: rework compiler encoding (add a compiler ID)

lib/spack/spack/solver/asp.py

@@ -818,6 +818,9 @@ def __init__(self, tests=False):
         self.compiler_version_constraints = set()
         self.post_facts = []
 
+        # (ID, CompilerSpec) -> dictionary of attributes
+        self.compiler_info = collections.defaultdict(dict)
+
         # hashes we've already added facts for
         self.seen_hashes = set()
         self.reusable_and_possible = {}
@@ -945,36 +948,59 @@ def conflict_rules(self, pkg):
     def compiler_facts(self):
         """Facts about available compilers."""
 
+        # Build a dictionary mapping a compiler spec to supported OS and targets
+        for entry in spack.compilers.all_compilers_config():
+            compiler_entry = entry["compiler"]
+            key = spack.spec.CompilerSpec(compiler_entry["spec"])
+
+            # FIXME: same spec with multiple OS and targets?
+            if key in self.compiler_info:
+                warnings.warn(
+                    f"compiler {key} has duplicate entries, only the first one will be considered"
+                )
+                continue
+
+            self.compiler_info[key]["os"] = compiler_entry["operating_system"]
+            self.compiler_info[key]["target"] = compiler_entry.get("target", None)
+            self.compiler_info[key]["compiler_obj"] = spack.compilers.compiler_from_dict(
+                compiler_entry
+            )
+
         self.gen.h2("Available compilers")
-        for compiler_id, compiler in enumerate(self.possible_compilers):
+        indexed_possible_compilers = list(enumerate(self.possible_compilers))
-            self.gen.fact(fn.compiler_version(compiler.name, compiler.version))
+        for compiler_id, compiler in indexed_possible_compilers:
+            self.gen.fact(fn.compiler_id(compiler_id))
+            self.gen.fact(fn.compiler_name(compiler_id, compiler.name))
+            self.gen.fact(fn.compiler_version(compiler_id, compiler.version))
+
+            compiler_entry = self.compiler_info[compiler]
+            # Use "get" to allow concretizing even when copiler existence checks are disabled
+            operating_system = compiler_entry.get("os")
+            if operating_system:
+                self.gen.fact(fn.compiler_os(compiler_id, operating_system))
+
+            target = compiler_entry.get("target")
+            if target is not None:
+                self.gen.fact(fn.compiler_target(compiler_id, target))
+
+            compiler_obj = compiler_entry.get("compiler_obj")
+            if compiler_obj:
+                for flag_type, flags in compiler_obj.flags.items():
+                    for flag in flags:
+                        self.gen.fact(fn.compiler_flag(compiler_id, flag_type, flag))
+
             self.gen.newline()
 
         # Set compiler defaults, given a list of possible compilers
-        self.gen.h2("Default compiler preferences")
+        self.gen.h2("Default compiler preferences (CompilerID, Weight)")
 
         ppk = spack.package_prefs.PackagePrefs("all", "compiler", all=False)
-        matches = sorted(self.possible_compilers, key=ppk)
+        matches = sorted(indexed_possible_compilers, key=lambda x: ppk(x[1]))
 
-        for i, cspec in enumerate(matches):
+        for weight, (compiler_id, cspec) in enumerate(matches):
-            f = fn.default_compiler_preference(cspec.name, cspec.version, i)
+            f = fn.default_compiler_preference(compiler_id, weight)
             self.gen.fact(f)
 
-        # Enumerate target families. This may be redundant, but compilers with
-        # custom versions will be able to concretize properly.
-        for entry in spack.compilers.all_compilers_config():
-            compiler_entry = entry["compiler"]
-            cspec = spack.spec.CompilerSpec(compiler_entry["spec"])
-            operating_system = compiler_entry["operating_system"]
-            self.gen.fact(fn.compiler_supports_os(cspec.name, cspec.version, operating_system))
-
-            if not compiler_entry.get("target", None):
-                continue
-
-            self.gen.fact(
-                fn.compiler_supports_target(cspec.name, cspec.version, compiler_entry["target"])
-            )
-
     def package_compiler_defaults(self, pkg):
         """Facts about packages' compiler prefs."""
 
@@ -1377,28 +1403,6 @@ def target_preferences(self, pkg_name):
                 fn.target_weight(pkg_name, str(preferred.architecture.target), i + offset)
             )
 
-    def flag_defaults(self):
-        self.gen.h2("Compiler flag defaults")
-
-        # types of flags that can be on specs
-        for flag in spack.spec.FlagMap.valid_compiler_flags():
-            self.gen.fact(fn.flag_type(flag))
-        self.gen.newline()
-
-        # flags from compilers.yaml
-        compilers = all_compilers_in_config()
-        seen = set()
-        for compiler in compilers:
-            # if there are multiple with the same spec, only use the first
-            if compiler.spec in seen:
-                continue
-            seen.add(compiler.spec)
-            for name, flags in compiler.flags.items():
-                for flag in flags:
-                    self.gen.fact(
-                        fn.compiler_version_flag(compiler.name, compiler.version, name, flag)
-                    )
-
     def spec_clauses(self, *args, **kwargs):
         """Wrap a call to `_spec_clauses()` into a try/except block that
         raises a comprehensible error message in case of failure.
@@ -1755,8 +1759,6 @@ def target_defaults(self, specs):
         if granularity == "generic":
             candidate_targets = [t for t in candidate_targets if t.vendor == "generic"]
 
-        compilers = self.possible_compilers
-
         # Add targets explicitly requested from specs
         for spec in specs:
             if not spec.architecture or not spec.architecture.target:
@@ -1774,7 +1776,7 @@ def target_defaults(self, specs):
                         candidate_targets.append(ancestor)
 
         best_targets = set([uarch.family.name])
-        for compiler in sorted(compilers):
+        for compiler_id, compiler in enumerate(self.possible_compilers):
             supported = self._supported_targets(compiler.name, compiler.version, candidate_targets)
 
             # If we can't find supported targets it may be due to custom
@@ -1793,13 +1795,10 @@ def target_defaults(self, specs):
 
             for target in supported:
                 best_targets.add(target.name)
-                self.gen.fact(
+                self.gen.fact(fn.compiler_supports_target(compiler_id, target.name))
-                    fn.compiler_supports_target(compiler.name, compiler.version, target.name)
-                )
 
-            self.gen.fact(
+            self.gen.fact(fn.compiler_supports_target(compiler_id, uarch.family.name))
-                fn.compiler_supports_target(compiler.name, compiler.version, uarch.family.name)
+            self.gen.newline()
-            )
 
         i = 0  # TODO compute per-target offset?
         for target in candidate_targets:
@@ -1859,7 +1858,6 @@ def generate_possible_compilers(self, specs):
                     # is already built
                 else:
                     cspecs.add(s.compiler)
-                    # FIXME (COMPILERS)
                     self.gen.fact(fn.allow_compiler(s.compiler.name, s.compiler.version))
 
         return list(
@@ -1921,14 +1919,12 @@ def versions_for(v):
                 self.possible_versions[pkg_name].add(version)
 
     def define_compiler_version_constraints(self):
-        compiler_list = spack.compilers.all_compiler_specs()
-        compiler_list = list(sorted(set(compiler_list)))
         for constraint in sorted(self.compiler_version_constraints):
-            for compiler in compiler_list:
+            for compiler_id, compiler in enumerate(self.possible_compilers):
                 if compiler.satisfies(constraint):
                     self.gen.fact(
                         fn.compiler_version_satisfies(
-                            constraint.name, constraint.versions, compiler.version
+                            constraint.name, constraint.versions, compiler_id
                         )
                     )
         self.gen.newline()
@@ -2089,6 +2085,11 @@ def setup(self, driver, specs, reuse=None):
             for reusable_spec in reuse:
                 self._facts_from_concrete_spec(reusable_spec, possible)
 
+        self.gen.h1("Possible flags on nodes")
+        for flag in spack.spec.FlagMap.valid_compiler_flags():
+            self.gen.fact(fn.flag_type(flag))
+        self.gen.newline()
+
         self.gen.h1("General Constraints")
         self.compiler_facts()
 
@@ -2101,7 +2102,6 @@ def setup(self, driver, specs, reuse=None):
         self.provider_defaults()
         self.provider_requirements()
         self.external_packages()
-        self.flag_defaults()
 
         self.gen.h1("Package Constraints")
         for pkg in sorted(self.pkgs):
@@ -2282,7 +2282,7 @@ def reorder_flags(self):
         flags will appear last on the compile line, in the order they
         were specified.
 
-        The solver determines wihch flags are on nodes; this routine
+        The solver determines which flags are on nodes; this routine
         imposes order afterwards.
         """
         # reverse compilers so we get highest priority compilers that share a spec

lib/spack/spack/solver/concretize.lp

@@ -821,9 +821,10 @@ node_target_compatible(Package, Target)
 % can't use targets on node if the compiler for the node doesn't support them
 error(2, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Target, Compiler, Version)
   :- attr("node_target", Package, Target),
-     not compiler_supports_target(Compiler, Version, Target),
+     node_compiler(Package, CompilerID),
-     attr("node_compiler", Package, Compiler),
+     not compiler_supports_target(CompilerID, Target),
-     attr("node_compiler_version", Package, Compiler, Version),
+     compiler_name(CompilerID, Compiler),
+     compiler_version(CompilerID, Version),
      build(Package).
 
 % if a target is set explicitly, respect it
@@ -857,32 +858,44 @@ error(2, "'{0} target={1}' is not compatible with this machine", Package, Target
 %-----------------------------------------------------------------------------
 % Compiler semantics
 %-----------------------------------------------------------------------------
-compiler(Compiler) :- compiler_version(Compiler, _).
+% There must be only one compiler set per built node.
-
+{ node_compiler(Package, CompilerID) : compiler_id(CompilerID) } :-
-% There must be only one compiler set per built node. The compiler
-% is chosen among available versions.
-{ attr("node_compiler_version", Package, Compiler, Version) : compiler_version(Compiler, Version) } :-
     attr("node", Package),
     build(Package).
 
+% Infer the compiler that matches a reused node
+node_compiler(Package, CompilerID)
+  :- attr("node_compiler_version", Package, CompilerName, CompilerVersion),
+     attr("node", Package),
+     compiler_name(CompilerID, CompilerName),
+     compiler_version(CompilerID, CompilerVersion),
+     not build(Package).
+
+% Expand the internal attribute into "attr("node_compiler_version")
+attr("node_compiler_version", Package, CompilerName, CompilerVersion)
+  :- node_compiler(Package, CompilerID),
+     compiler_name(CompilerID, CompilerName),
+     compiler_version(CompilerID, CompilerVersion),
+     build(Package).
+
+attr("node_compiler", Package, CompilerName)
+  :- attr("node_compiler_version", Package, CompilerName, CompilerVersion).
+
 error(2, "No valid compiler version found for '{0}'", Package)
   :- attr("node", Package),
-     C = #count{ Version : attr("node_compiler_version", Package, _, Version)},
+     not attr("node_compiler_version", Package, _, _).
-     C < 1.
-error(2, "'{0}' compiler constraints '%{1}@{2}' and '%{3}@{4}' are incompatible", Package, Compiler1, Version1, Compiler2, Version2)
-  :- attr("node", Package),
-     attr("node_compiler_version", Package, Compiler1, Version1),
-     attr("node_compiler_version", Package, Compiler2, Version2),
-     (Compiler1, Version1) < (Compiler2, Version2). % see[1]
 
-% Sometimes we just need to know the compiler and not the version
+error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2)
-attr("node_compiler", Package, Compiler) :- attr("node_compiler_version", Package, Compiler, _).
+  :- attr("node", Package),
+     attr("node_compiler_version", Package, C1, V1),
+     attr("node_compiler_version", Package, C2, V2),
+     (C1, V1) < (C2, V2). % see[1]
 
 % We can't have a compiler be enforced and select the version from another compiler
 error(2, "Cannot concretize {0} with two compilers {1}@{2} and {3}@{4}", Package, C1, V1, C2, V2)
  :- attr("node_compiler_version", Package, C1, V1),
     attr("node_compiler_version", Package, C2, V2),
-    (C1, V1) != (C2, V2).
+    (C1, V1) < (C2, V2).
 
 error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Compiler1, Compiler2, Version)
  :- attr("node_compiler", Package, Compiler1),
@@ -893,37 +906,41 @@ error(2, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Co
 error(1, "No valid compiler for {0} satisfies '%{1}'", Package, Compiler)
   :- attr("node", Package),
      attr("node_compiler_version_satisfies", Package, Compiler, ":"),
-     C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, ":", Version) },
+     not compiler_version_satisfies(Compiler, ":", _).
-     C < 1.
 
 % If the compiler of a node must satisfy a constraint, then its version
 % must be chosen among the ones that satisfy said constraint
 error(2, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint)
   :- attr("node", Package),
      attr("node_compiler_version_satisfies", Package, Compiler, Constraint),
-     C = #count{ Version : attr("node_compiler_version", Package, Compiler, Version), compiler_version_satisfies(Compiler, Constraint, Version) },
+		 not compiler_version_satisfies(Compiler, Constraint, _).
-     C < 1.
 
 % If the node is associated with a compiler and the compiler satisfy a constraint, then
 % the compiler associated with the node satisfy the same constraint
 attr("node_compiler_version_satisfies", Package, Compiler, Constraint)
-  :- attr("node_compiler_version", Package, Compiler, Version),
+  :- node_compiler(Package, CompilerID),
-     compiler_version_satisfies(Compiler, Constraint, Version).
+     compiler_name(CompilerID, Compiler),
+     compiler_version_satisfies(Compiler, Constraint, CompilerID).
 
 #defined compiler_version_satisfies/3.
 
 % If the compiler version was set from the command line,
 % respect it verbatim
-attr("node_compiler_version", Package, Compiler, Version) :-
+:- attr("node_compiler_version_set", Package, Compiler, Version),
-     attr("node_compiler_version_set", Package, Compiler, Version).
+   not attr("node_compiler_version", Package, Compiler, Version).
+
+:- attr("node_compiler_set", Package, Compiler),
+   not attr("node_compiler_version", Package, Compiler, _).
 
 % Cannot select a compiler if it is not supported on the OS
 % Compilers that are explicitly marked as allowed
 % are excluded from this check
 error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler, Version, OS)
-  :- attr("node_compiler_version", Package, Compiler, Version),
+  :- attr("node_os", Package, OS),
-     attr("node_os", Package, OS),
+     node_compiler(Package, CompilerID),
-     not compiler_supports_os(Compiler, Version, OS),
+     compiler_name(CompilerID, Compiler),
+     compiler_version(CompilerID, Version),
+     not compiler_os(CompilerID, OS),
      not allow_compiler(Compiler, Version),
      build(Package).
 
@@ -931,8 +948,8 @@ error(2, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler
 % same compiler there's a mismatch.
 compiler_match(Package, Dependency)
   :- depends_on(Package, Dependency),
-     attr("node_compiler_version", Package, Compiler, Version),
+     node_compiler(Package, CompilerID),
-     attr("node_compiler_version", Dependency, Compiler, Version).
+     node_compiler(Dependency, CompilerID).
 
 compiler_mismatch(Package, Dependency)
   :- depends_on(Package, Dependency),
@@ -944,25 +961,32 @@ compiler_mismatch_required(Package, Dependency)
      attr("node_compiler_set", Dependency, _),
      not compiler_match(Package, Dependency).
 
-#defined compiler_supports_os/3.
+#defined compiler_os/3.
 #defined allow_compiler/2.
 
 % compilers weighted by preference according to packages.yaml
 compiler_weight(Package, Weight)
- :- attr("node_compiler_version", Package, Compiler, V),
+ :- node_compiler(Package, CompilerID),
+    compiler_name(CompilerID, Compiler),
+    compiler_version(CompilerID, V),
     node_compiler_preference(Package, Compiler, V, Weight).
 compiler_weight(Package, Weight)
- :- attr("node_compiler_version", Package, Compiler, V),
+ :- node_compiler(Package, CompilerID),
+    compiler_name(CompilerID, Compiler),
+    compiler_version(CompilerID, V),
     not node_compiler_preference(Package, Compiler, V, _),
-    default_compiler_preference(Compiler, V, Weight).
+    default_compiler_preference(CompilerID, Weight).
 compiler_weight(Package, 100)
- :- attr("node_compiler_version", Package, Compiler, Version),
+ :- node_compiler(Package, CompilerID),
-    not node_compiler_preference(Package, Compiler, Version, _),
+    compiler_name(CompilerID, Compiler),
-    not default_compiler_preference(Compiler, Version, _).
+    compiler_version(CompilerID, V),
+    not node_compiler_preference(Package, Compiler, V, _),
+    not default_compiler_preference(CompilerID, _).
 
 % For the time being, be strict and reuse only if the compiler match one we have on the system
 error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missing_compilers:true if intended.", Package, Compiler, Version)
- :- attr("node_compiler_version", Package, Compiler, Version), not compiler_version(Compiler, Version).
+ :- attr("node_compiler_version", Package, Compiler, Version),
+    not node_compiler(Package, _).
 
 #defined node_compiler_preference/4.
 #defined default_compiler_preference/3.
@@ -974,10 +998,11 @@ error(2, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missin
 % propagate flags when compiler match
 can_inherit_flags(Package, Dependency, FlagType)
  :- depends_on(Package, Dependency),
-    attr("node_compiler", Package, Compiler),
+    node_compiler(Package, CompilerID),
-    attr("node_compiler", Dependency, Compiler),
+    node_compiler(Dependency, CompilerID),
     not attr("node_flag_set", Dependency, FlagType, _),
-    compiler(Compiler), flag_type(FlagType).
+    compiler_id(CompilerID),
+    flag_type(FlagType).
 
 node_flag_inherited(Dependency, FlagType, Flag)
  :- attr("node_flag_set", Package, FlagType, Flag), can_inherit_flags(Package, Dependency, FlagType),
@@ -1004,19 +1029,21 @@ attr("node_flag_source", Dependency, FlagType, Q)
 
 % compiler flags from compilers.yaml are put on nodes if compiler matches
 attr("node_flag", Package, FlagType, Flag)
- :- compiler_version_flag(Compiler, Version, FlagType, Flag),
+ :- compiler_flag(CompilerID, FlagType, Flag),
-    attr("node_compiler_version", Package, Compiler, Version),
+    node_compiler(Package, CompilerID),
     flag_type(FlagType),
-    compiler(Compiler),
+    compiler_id(CompilerID),
-    compiler_version(Compiler, Version).
+    compiler_name(CompilerID, CompilerName),
+    compiler_version(CompilerID, Version).
 
 attr("node_flag_compiler_default", Package)
  :- not attr("node_flag_set", Package, FlagType, _),
-    compiler_version_flag(Compiler, Version, FlagType, Flag),
+    compiler_flag(CompilerID, FlagType, Flag),
-    attr("node_compiler_version", Package, Compiler, Version),
+    node_compiler(Package, CompilerID),
     flag_type(FlagType),
-    compiler(Compiler),
+    compiler_id(CompilerID),
-    compiler_version(Compiler, Version).
+    compiler_name(CompilerID, CompilerName),
+    compiler_version(CompilerID, Version).
 
 % if a flag is set to something or inherited, it's included
 attr("node_flag", Package, FlagType, Flag) :- attr("node_flag_set", Package, FlagType, Flag).
@@ -1027,7 +1054,7 @@ attr("node_flag", Package, FlagType, Flag)
 attr("no_flags", Package, FlagType)
  :- not attr("node_flag", Package, FlagType, _), attr("node", Package), flag_type(FlagType).
 
-#defined compiler_version_flag/4.
+#defined compiler_flag/3.
 
 
 %-----------------------------------------------------------------------------
@@ -1308,6 +1335,7 @@ opt_criterion(5, "non-preferred targets").
 #heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
 #heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
 #heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true]
+#heuristic attr("node_compiler_version", Package, Compiler, Version) : default_compiler_preference(ID, 0), compiler_name(ID, Compiler), compiler_version(ID, Version), attr("node", Package). [10, true]
 
 %-----------
 % Notes