Compilers can inject first order rules into the solver

* Restore PackageBase class, and modify only ASP

  This prevents a noticeable slowdown in concretization
  due to the number of directives involved.

* Fix issue with 'clang' being preferred to 'gcc',
  due to runtime version weights

* Constraints on runtimes are declared by compilers

  The declaration of available runtime versions, and of
  their compatibility constraints are in the associated
  compiler class.

Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>

lib/spack/spack/main.py

@@ -36,7 +36,6 @@
 import spack.config
 import spack.environment as ev
 import spack.modules
-import spack.package_base
 import spack.paths
 import spack.platforms
 import spack.repo
@@ -608,7 +607,6 @@ def setup_main_options(args):
             [(key, [spack.paths.mock_packages_path])]
         )
         spack.repo.PATH = spack.repo.create(spack.config.CONFIG)
-        spack.package_base.WITH_GCC_RUNTIME = False
 
     # If the user asked for it, don't check ssl certs.
     if args.insecure:

lib/spack/spack/package_base.py

@@ -53,7 +53,6 @@
 import spack.util.environment
 import spack.util.path
 import spack.util.web
-from spack.directives import _depends_on
 from spack.filesystem_view import YamlFilesystemView
 from spack.install_test import (
     PackageTest,
@@ -77,7 +76,6 @@
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
 
-WITH_GCC_RUNTIME = True
 
 #: Filename for the Spack build/install log.
 _spack_build_logfile = "spack-build-out.txt"
@@ -373,20 +371,6 @@ def _wrapper(instance, *args, **kwargs):
     return _execute_under_condition
 
 
-class BinaryPackage:
-    """This adds a universal dependency on gcc-runtime."""
-
-    def maybe_depend_on_gcc_runtime(self):
-        # Do not depend on itself, and allow tests to disable this universal dep
-        if self.name == "gcc-runtime" or not WITH_GCC_RUNTIME:
-            return
-        for v in ["13", "12", "11", "10", "9", "8", "7", "6", "5", "4"]:
-            _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=linux")
-            _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=cray")
-
-    _directives_to_be_executed = [maybe_depend_on_gcc_runtime]
-
-
 class PackageViewMixin:
     """This collects all functionality related to adding installed Spack
     package to views. Packages can customize how they are added to views by
@@ -449,7 +433,7 @@ def remove_files_from_view(self, view, merge_map):
 Pb = TypeVar("Pb", bound="PackageBase")
 
 
-class PackageBase(WindowsRPath, PackageViewMixin, BinaryPackage, metaclass=PackageMeta):
+class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
     """This is the superclass for all spack packages.
 
     ***The Package class***

lib/spack/spack/solver/asp.py

@@ -11,6 +11,7 @@
 import pathlib
 import pprint
 import re
+import sys
 import types
 import warnings
 from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union
@@ -61,6 +62,8 @@
 ASTType = None
 parse_files = None
 
+#: Enable the addition of a runtime node
+WITH_RUNTIME = sys.platform != "win32"
 
 #: Data class that contain configuration on what a
 #: clingo solve should output.
@@ -122,6 +125,8 @@ class Provenance(enum.IntEnum):
     PACKAGE_PY = enum.auto()
     # An installed spec
     INSTALLED = enum.auto()
+    # A runtime injected from another package (e.g. a compiler)
+    RUNTIME = enum.auto()
 
     def __str__(self):
         return f"{self._name_.lower()}"
@@ -2023,7 +2028,9 @@ class Body:
                     f.node_compiler_version(spec.name, spec.compiler.name, spec.compiler.version)
                 )
 
-            elif spec.compiler.versions:
+            elif spec.compiler.versions and spec.compiler.versions != vn.any_version:
+                # The condition above emits a facts only if we have an actual constraint
+                # on the compiler version, and avoids emitting them if any version is fine
                 clauses.append(
                     fn.attr(
                         "node_compiler_version_satisfies",
@@ -2578,6 +2585,9 @@ def setup(
         self.possible_virtuals = node_counter.possible_virtuals()
         self.pkgs = node_counter.possible_dependencies()
 
+        runtimes = spack.repo.PATH.packages_with_tags("runtime")
+        self.pkgs.update(set(runtimes))
+
         # Fail if we already know an unreachable node is requested
         for spec in specs:
             missing_deps = [
@@ -2678,6 +2688,10 @@ def setup(
         self.gen.h1("Variant Values defined in specs")
         self.define_variant_values()
 
+        if WITH_RUNTIME:
+            self.gen.h1("Runtimes")
+            self.define_runtime_constraints()
+
         self.gen.h1("Version Constraints")
         self.collect_virtual_constraints()
         self.define_version_constraints()
@@ -2688,6 +2702,21 @@ def setup(
         self.gen.h1("Target Constraints")
         self.define_target_constraints()
 
+    def define_runtime_constraints(self):
+        """Define the constraints to be imposed on the runtimes"""
+        recorder = RuntimePropertyRecorder(self)
+        for compiler in self.possible_compilers:
+            if compiler.name != "gcc":
+                continue
+            try:
+                compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name)
+            except spack.repo.UnknownPackageError:
+                continue
+            if hasattr(compiler_cls, "runtime_constraints"):
+                compiler_cls.runtime_constraints(compiler=compiler, pkg=recorder)
+
+        recorder.consume_facts()
+
     def literal_specs(self, specs):
         for spec in specs:
             self.gen.h2("Spec: %s" % str(spec))
@@ -2796,6 +2825,157 @@ def _specs_from_requires(self, pkg_name, section):
                 yield _spec_with_default_name(s, pkg_name)
 
 
+class RuntimePropertyRecorder:
+    """An object of this class is injected in callbacks to compilers, to let them declare
+    properties of the runtimes they support and of the runtimes they provide, and to add
+    runtime dependencies to the nodes using said compiler.
+
+    The usage of the object is the following. First, a runtime package name or the wildcard
+    "*" are passed as an argument to __call__, to set which kind of package we are referring to.
+    Then we can call one method with a directive-like API.
+
+    Examples:
+        >>> pkg = RuntimePropertyRecorder(setup)
+        >>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime'
+        >>> pkg("*").depends_on(
+        ...     "gcc-runtime",
+        ...     when="%gcc",
+        ...     type="link",
+        ...     description="If any package uses %gcc, it depends on gcc-runtime"
+        ... )
+        >>> # The version of gcc-runtime is the same as the %gcc used to "compile" it
+        >>> pkg("gcc-runtime").requires("@=9.4.0", when="%gcc@=9.4.0")
+    """
+
+    def __init__(self, setup):
+        self._setup = setup
+        self.rules = []
+        self.runtime_conditions = set()
+        # State of this object set in the __call__ method, and reset after
+        # each directive-like method
+        self.current_package = None
+
+    def __call__(self, package_name: str) -> "RuntimePropertyRecorder":
+        """Sets a package name for the next directive-like method call"""
+        assert self.current_package is None, f"state was already set to '{self.current_package}'"
+        self.current_package = package_name
+        return self
+
+    def reset(self):
+        """Resets the current state."""
+        self.current_package = None
+
+    def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None:
+        """Injects conditional dependencies on packages.
+
+        Args:
+            dependency_str: the dependency spec to inject
+            when: anonymous condition to be met on a package to have the dependency
+            type: dependency type
+            description: human-readable description of the rule for adding the dependency
+        """
+        # TODO: The API for this function is not final, and is still subject to change. At
+        # TODO: the moment, we implemented only the features strictly needed for the
+        # TODO: functionality currently provided by Spack, and we assert nothing else is required.
+        msg = "the 'depends_on' method can be called only with pkg('*')"
+        assert self.current_package == "*", msg
+
+        when_spec = spack.spec.Spec(when)
+        assert when_spec.name is None, "only anonymous when specs are accepted"
+
+        dependency_spec = spack.spec.Spec(dependency_str)
+        if dependency_spec.versions != vn.any_version:
+            self._setup.version_constraints.add((dependency_spec.name, dependency_spec.versions))
+
+        placeholder = "XXX"
+        node_variable = "node(ID, Package)"
+        when_spec.name = placeholder
+
+        body_clauses = self._setup.spec_clauses(when_spec, body=True)
+        body_str = (
+            f"  {f',{os.linesep}  '.join(str(x) for x in body_clauses)},\n"
+            f"  not runtime(Package)"
+        ).replace(f'"{placeholder}"', f"{node_variable}")
+        head_clauses = self._setup.spec_clauses(dependency_spec, body=False)
+
+        runtime_pkg = dependency_spec.name
+        main_rule = (
+            f"% {description}\n"
+            f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :'
+            f' max_dupes("gcc-runtime", X)}} 1:-\n'
+            f"{body_str}.\n\n"
+        )
+        self.rules.append(main_rule)
+        for clause in head_clauses:
+            if clause.args[0] == "node":
+                continue
+            runtime_node = f'node(RuntimeID, "{runtime_pkg}")'
+            head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node)
+            rule = (
+                f"{head_str} :-\n"
+                f'  attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n'
+                f"{body_str}.\n\n"
+            )
+            self.rules.append(rule)
+
+        self.reset()
+
+    def requires(self, impose: str, *, when: str):
+        """Injects conditional requirements on a given package.
+
+        Args:
+            impose: constraint to be imposed
+            when: condition triggering the constraint
+        """
+        msg = "the 'requires' method cannot be called with pkg('*') or without setting the package"
+        assert self.current_package is not None and self.current_package != "*", msg
+
+        imposed_spec = spack.spec.Spec(f"{self.current_package}{impose}")
+        when_spec = spack.spec.Spec(f"{self.current_package}{when}")
+
+        assert imposed_spec.versions.concrete, f"{impose} must have a concrete version"
+        assert when_spec.compiler.concrete, f"{when} must have a concrete compiler"
+
+        # Add versions to possible versions
+        for s in (imposed_spec, when_spec):
+            if not s.versions.concrete:
+                continue
+            self._setup.possible_versions[s.name].add(s.version)
+            self._setup.declared_versions[s.name].append(
+                DeclaredVersion(version=s.version, idx=0, origin=Provenance.RUNTIME)
+            )
+
+        self.runtime_conditions.add((imposed_spec, when_spec))
+        self.reset()
+
+    def consume_facts(self):
+        """Consume the facts collected by this object, and emits rules and
+        facts for the runtimes.
+        """
+        self._setup.gen.h2("Runtimes: rules")
+        self._setup.gen.newline()
+        for rule in self.rules:
+            if not isinstance(self._setup.gen.out, llnl.util.lang.Devnull):
+                self._setup.gen.out.write(rule)
+            self._setup.gen.control.add("base", [], rule)
+
+        self._setup.gen.h2("Runtimes: conditions")
+        for runtime_pkg in spack.repo.PATH.packages_with_tags("runtime"):
+            self._setup.gen.fact(fn.runtime(runtime_pkg))
+            self._setup.gen.fact(fn.possible_in_link_run(runtime_pkg))
+            self._setup.gen.newline()
+            # Inject version rules for runtimes (versions are declared based
+            # on the available compilers)
+            self._setup.pkg_version_rules(runtime_pkg)
+
+        for imposed_spec, when_spec in self.runtime_conditions:
+            msg = f"{when_spec} requires {imposed_spec} at runtime"
+            _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg)
+
+        self._setup.trigger_rules()
+        self._setup.effect_rules()
+
+
 class SpecBuilder:
     """Class with actions to rebuild a spec from ASP results."""
 

lib/spack/spack/test/concretize_compiler_runtimes.py

@@ -0,0 +1,42 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+import pytest
+
+import spack.paths
+import spack.repo
+import spack.solver.asp
+import spack.spec
+from spack.version import Version
+
+pytestmark = [pytest.mark.only_clingo("Original concretizer does not support compiler runtimes")]
+
+
+@pytest.fixture
+def runtime_repo(config):
+    repo = os.path.join(spack.paths.repos_path, "compiler_runtime.test")
+    with spack.repo.use_repositories(repo) as mock_repo:
+        yield mock_repo
+
+
+@pytest.fixture
+def enable_runtimes():
+    original = spack.solver.asp.WITH_RUNTIME
+    spack.solver.asp.WITH_RUNTIME = True
+    yield
+    spack.solver.asp.WITH_RUNTIME = original
+
+
+def test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo, enable_runtimes):
+    s = spack.spec.Spec("a%gcc@10.2.1 ^b%gcc@4.5.0").concretized()
+    a, b = s["a"], s["b"]
+
+    # Both a and b should depend on the same gcc-runtime directly
+    assert a.dependencies("gcc-runtime") == b.dependencies("gcc-runtime")
+
+    # And the gcc-runtime version should be that of the newest gcc used in the dag.
+    assert a["gcc-runtime"].version == Version("10.2.1")

lib/spack/spack/test/conftest.py

@@ -44,6 +44,7 @@
 import spack.paths
 import spack.platforms
 import spack.repo
+import spack.solver.asp
 import spack.stage
 import spack.store
 import spack.subprocess_context
@@ -57,11 +58,6 @@
 from spack.util.pattern import Bunch
 
 
-@pytest.fixture(scope="session", autouse=True)
-def drop_gcc_runtime():
-    spack.package_base.WITH_GCC_RUNTIME = False
-
-
 def ensure_configuration_fixture_run_before(request):
     """Ensure that fixture mutating the configuration run before the one where
     the function is called.

var/spack/repos/builtin/packages/gcc-runtime/package.py

@@ -20,6 +20,8 @@ class GccRuntime(Package):
     homepage = "https://gcc.gnu.org"
     has_code = False
 
+    tags = ["runtime"]
+
     maintainers("haampie")
 
     license("GPL-3.0-or-later WITH GCC-exception-3.1")
@@ -42,76 +44,6 @@ class GccRuntime(Package):
         "ubsan",
     ]
 
-    for v in [
-        "13.2",
-        "13.1",
-        "12.3",
-        "12.2",
-        "12.1",
-        "11.4",
-        "11.3",
-        "11.2",
-        "11.1",
-        "10.5",
-        "10.4",
-        "10.3",
-        "10.2",
-        "10.1",
-        "9.5",
-        "9.4",
-        "9.3",
-        "9.2",
-        "9.1",
-        "8.5",
-        "8.4",
-        "8.3",
-        "8.2",
-        "8.1",
-        "7.5",
-        "7.4",
-        "7.3",
-        "7.2",
-        "7.1",
-        "6.5",
-        "6.4",
-        "6.3",
-        "6.2",
-        "6.1",
-        "5.5",
-        "5.4",
-        "5.3",
-        "5.2",
-        "5.1",
-        "4.9.4",
-        "4.9.3",
-        "4.9.2",
-        "4.9.1",
-        "4.9.0",
-        "4.8.5",
-        "4.8.4",
-        "4.8.3",
-        "4.8.2",
-        "4.8.1",
-        "4.8.0",
-        "4.7.4",
-        "4.7.3",
-        "4.7.2",
-        "4.7.1",
-        "4.7.0",
-        "4.6.4",
-        "4.6.3",
-        "4.6.2",
-        "4.6.1",
-        "4.6.0",
-        "4.5.4",
-        "4.5.3",
-        "4.5.2",
-        "4.5.1",
-        "4.5.0",
-    ]:
-        version(v)
-        requires(f"%gcc@{v}", when=f"@{v}")
-
     def install(self, spec, prefix):
         if spec.platform in ["linux", "cray", "freebsd"]:
             libraries = self._get_libraries_elf()

var/spack/repos/builtin/packages/gcc/package.py

@@ -1111,3 +1111,32 @@ def detect_gdc(self):
                         ),
                     ),
                 )
+
+    @classmethod
+    def runtime_constraints(cls, *, compiler, pkg):
+        """Callback function to inject runtime-related rules into the solver.
+
+        Rule-injection is obtained through method calls of the ``pkg`` argument.
+
+        Documentation for this function is temporary. When the API will be in its final state,
+        we'll document the behavior at https://spack.readthedocs.io/en/latest/
+
+        Args:
+            compiler: compiler object (node attribute) currently considered
+            pkg: object used to forward information to the solver
+        """
+        pkg("*").depends_on(
+            "gcc-runtime",
+            when="%gcc",
+            type="link",
+            description="If any package uses %gcc, it depends on gcc-runtime",
+        )
+        pkg("*").depends_on(
+            f"gcc-runtime@{str(compiler.version)}:",
+            when=f"%{str(compiler.spec)}",
+            type="link",
+            description=f"If any package uses %{str(compiler.spec)}, "
+            f"it depends on gcc-runtime@{str(compiler.version)}:",
+        )
+        # The version of gcc-runtime is the same as the %gcc used to "compile" it
+        pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")

var/spack/repos/compiler_runtime.test/packages/a/package.py

@@ -0,0 +1,13 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class A(Package):
+    homepage = "http://www.example.com"
+    has_code = False
+
+    version("1.0")
+    depends_on("b")

var/spack/repos/compiler_runtime.test/packages/b/package.py

@@ -0,0 +1,12 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class B(Package):
+    homepage = "http://www.example.com"
+    has_code = False
+
+    version("1.0")

var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py

@@ -0,0 +1,13 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class GccRuntime(Package):
+    homepage = "https://example.com"
+    has_code = False
+    tags = ["runtime"]
+    requires("%gcc")

var/spack/repos/compiler_runtime.test/packages/gcc/package.py

@@ -0,0 +1,32 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Gcc(Package):
+    homepage = "http://www.example.com/"
+    has_code = False
+
+    version("13.2.0")
+    version("12.3.0")
+
+    @classmethod
+    def runtime_constraints(cls, *, compiler, pkg):
+        pkg("*").depends_on(
+            "gcc-runtime",
+            when="%gcc",
+            type="link",
+            description="If any package uses %gcc, it depends on gcc-runtime",
+        )
+        pkg("*").depends_on(
+            f"gcc-runtime@{str(compiler.version)}:",
+            when=f"%{str(compiler.spec)}",
+            type="link",
+            description=f"If any package uses %{str(compiler.spec)}, "
+            f"it depends on gcc-runtime@{str(compiler.version)}:",
+        )
+        # The version of gcc-runtime is the same as the %gcc used to "compile" it
+        pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")

var/spack/repos/compiler_runtime.test/repo.yaml

@@ -0,0 +1,2 @@
+repo:
+  namespace: compiler_runtime.test