zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compilers can inject first order rules into the solver

Browse Source 
* Restore PackageBase class, and modify only ASP

  This prevents a noticeable slowdown in concretization
  due to the number of directives involved.

* Fix issue with 'clang' being preferred to 'gcc',
  due to runtime version weights

* Constraints on runtimes are declared by compilers

  The declaration of available runtime versions, and of
  their compatibility constraints are in the associated
  compiler class.

Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>
This commit is contained in:
Massimiliano Culpo 2023-11-30 18:36:24 +01:00 committed by Todd Gamblin
parent 8371bb4e19
commit ea7e3e4f9f
12 changed files with 328 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/main.py
View File

@ -36,7 +36,6 @@
import spack.config
import spack.environment as ev
import spack.modules
import spack.package_base
import spack.paths
import spack.platforms
import spack.repo
@ -608,7 +607,6 @@ def setup_main_options(args):
[(key, [spack.paths.mock_packages_path])]
)
spack.repo.PATH = spack.repo.create(spack.config.CONFIG)
spack.package_base.WITH_GCC_RUNTIME = False
# If the user asked for it, don't check ssl certs.
if args.insecure:

18
lib/spack/spack/package_base.py
View File

@ -53,7 +53,6 @@
import spack.util.environment
import spack.util.path
import spack.util.web
from spack.directives import _depends_on
from spack.filesystem_view import YamlFilesystemView
from spack.install_test import (
PackageTest,
@ -77,7 +76,6 @@
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
WITH_GCC_RUNTIME = True
#: Filename for the Spack build/install log.
_spack_build_logfile = "spack-build-out.txt"
@ -373,20 +371,6 @@ def _wrapper(instance, *args, **kwargs):
return _execute_under_condition
class BinaryPackage:
"""This adds a universal dependency on gcc-runtime."""
def maybe_depend_on_gcc_runtime(self):
# Do not depend on itself, and allow tests to disable this universal dep
if self.name == "gcc-runtime" or not WITH_GCC_RUNTIME:
return
for v in ["13", "12", "11", "10", "9", "8", "7", "6", "5", "4"]:
_depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=linux")
_depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=cray")
_directives_to_be_executed = [maybe_depend_on_gcc_runtime]
class PackageViewMixin:
"""This collects all functionality related to adding installed Spack
package to views. Packages can customize how they are added to views by
@ -449,7 +433,7 @@ def remove_files_from_view(self, view, merge_map):
Pb = TypeVar("Pb", bound="PackageBase")
class PackageBase(WindowsRPath, PackageViewMixin, BinaryPackage, metaclass=PackageMeta):
class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
"""This is the superclass for all spack packages.
***The Package class***

182
lib/spack/spack/solver/asp.py
View File

@ -11,6 +11,7 @@
import pathlib
import pprint
import re
import sys
import types
import warnings
from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union
@ -61,6 +62,8 @@
ASTType = None
parse_files = None
#: Enable the addition of a runtime node
WITH_RUNTIME = sys.platform != "win32"
#: Data class that contain configuration on what a
#: clingo solve should output.
@ -122,6 +125,8 @@ class Provenance(enum.IntEnum):
PACKAGE_PY = enum.auto()
# An installed spec
INSTALLED = enum.auto()
# A runtime injected from another package (e.g. a compiler)
RUNTIME = enum.auto()
def __str__(self):
return f"{self._name_.lower()}"
@ -2023,7 +2028,9 @@ class Body:
f.node_compiler_version(spec.name, spec.compiler.name, spec.compiler.version)
)
elif spec.compiler.versions:
elif spec.compiler.versions and spec.compiler.versions != vn.any_version:
# The condition above emits a facts only if we have an actual constraint
# on the compiler version, and avoids emitting them if any version is fine
clauses.append(
fn.attr(
"node_compiler_version_satisfies",
@ -2578,6 +2585,9 @@ def setup(
self.possible_virtuals = node_counter.possible_virtuals()
self.pkgs = node_counter.possible_dependencies()
runtimes = spack.repo.PATH.packages_with_tags("runtime")
self.pkgs.update(set(runtimes))
# Fail if we already know an unreachable node is requested
for spec in specs:
missing_deps = [
@ -2678,6 +2688,10 @@ def setup(
self.gen.h1("Variant Values defined in specs")
self.define_variant_values()
if WITH_RUNTIME:
self.gen.h1("Runtimes")
self.define_runtime_constraints()
self.gen.h1("Version Constraints")
self.collect_virtual_constraints()
self.define_version_constraints()
@ -2688,6 +2702,21 @@ def setup(
self.gen.h1("Target Constraints")
self.define_target_constraints()
def define_runtime_constraints(self):
"""Define the constraints to be imposed on the runtimes"""
recorder = RuntimePropertyRecorder(self)
for compiler in self.possible_compilers:
if compiler.name != "gcc":
continue
try:
compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name)
except spack.repo.UnknownPackageError:
continue
if hasattr(compiler_cls, "runtime_constraints"):
compiler_cls.runtime_constraints(compiler=compiler, pkg=recorder)
recorder.consume_facts()
def literal_specs(self, specs):
for spec in specs:
self.gen.h2("Spec: %s" % str(spec))
@ -2796,6 +2825,157 @@ def _specs_from_requires(self, pkg_name, section):
yield _spec_with_default_name(s, pkg_name)
class RuntimePropertyRecorder:
"""An object of this class is injected in callbacks to compilers, to let them declare
properties of the runtimes they support and of the runtimes they provide, and to add
runtime dependencies to the nodes using said compiler.
The usage of the object is the following. First, a runtime package name or the wildcard
"*" are passed as an argument to __call__, to set which kind of package we are referring to.
Then we can call one method with a directive-like API.
Examples:
>>> pkg = RuntimePropertyRecorder(setup)
>>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime'
>>> pkg("*").depends_on(
... "gcc-runtime",
... when="%gcc",
... type="link",
... description="If any package uses %gcc, it depends on gcc-runtime"
... )
>>> # The version of gcc-runtime is the same as the %gcc used to "compile" it
>>> pkg("gcc-runtime").requires("@=9.4.0", when="%gcc@=9.4.0")
"""
def __init__(self, setup):
self._setup = setup
self.rules = []
self.runtime_conditions = set()
# State of this object set in the __call__ method, and reset after
# each directive-like method
self.current_package = None
def __call__(self, package_name: str) -> "RuntimePropertyRecorder":
"""Sets a package name for the next directive-like method call"""
assert self.current_package is None, f"state was already set to '{self.current_package}'"
self.current_package = package_name
return self
def reset(self):
"""Resets the current state."""
self.current_package = None
def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None:
"""Injects conditional dependencies on packages.
Args:
dependency_str: the dependency spec to inject
when: anonymous condition to be met on a package to have the dependency
type: dependency type
description: human-readable description of the rule for adding the dependency
"""
# TODO: The API for this function is not final, and is still subject to change. At
# TODO: the moment, we implemented only the features strictly needed for the
# TODO: functionality currently provided by Spack, and we assert nothing else is required.
msg = "the 'depends_on' method can be called only with pkg('*')"
assert self.current_package == "*", msg
when_spec = spack.spec.Spec(when)
assert when_spec.name is None, "only anonymous when specs are accepted"
dependency_spec = spack.spec.Spec(dependency_str)
if dependency_spec.versions != vn.any_version:
self._setup.version_constraints.add((dependency_spec.name, dependency_spec.versions))
placeholder = "XXX"
node_variable = "node(ID, Package)"
when_spec.name = placeholder
body_clauses = self._setup.spec_clauses(when_spec, body=True)
body_str = (
f" {f',{os.linesep} '.join(str(x) for x in body_clauses)},\n"
f" not runtime(Package)"
).replace(f'"{placeholder}"', f"{node_variable}")
head_clauses = self._setup.spec_clauses(dependency_spec, body=False)
runtime_pkg = dependency_spec.name
main_rule = (
f"% {description}\n"
f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :'
f' max_dupes("gcc-runtime", X)}} 1:-\n'
f"{body_str}.\n\n"
)
self.rules.append(main_rule)
for clause in head_clauses:
if clause.args[0] == "node":
continue
runtime_node = f'node(RuntimeID, "{runtime_pkg}")'
head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node)
rule = (
f"{head_str} :-\n"
f' attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n'
f"{body_str}.\n\n"
)
self.rules.append(rule)
self.reset()
def requires(self, impose: str, *, when: str):
"""Injects conditional requirements on a given package.
Args:
impose: constraint to be imposed
when: condition triggering the constraint
"""
msg = "the 'requires' method cannot be called with pkg('*') or without setting the package"
assert self.current_package is not None and self.current_package != "*", msg
imposed_spec = spack.spec.Spec(f"{self.current_package}{impose}")
when_spec = spack.spec.Spec(f"{self.current_package}{when}")
assert imposed_spec.versions.concrete, f"{impose} must have a concrete version"
assert when_spec.compiler.concrete, f"{when} must have a concrete compiler"
# Add versions to possible versions
for s in (imposed_spec, when_spec):
if not s.versions.concrete:
continue
self._setup.possible_versions[s.name].add(s.version)
self._setup.declared_versions[s.name].append(
DeclaredVersion(version=s.version, idx=0, origin=Provenance.RUNTIME)
)
self.runtime_conditions.add((imposed_spec, when_spec))
self.reset()
def consume_facts(self):
"""Consume the facts collected by this object, and emits rules and
facts for the runtimes.
"""
self._setup.gen.h2("Runtimes: rules")
self._setup.gen.newline()
for rule in self.rules:
if not isinstance(self._setup.gen.out, llnl.util.lang.Devnull):
self._setup.gen.out.write(rule)
self._setup.gen.control.add("base", [], rule)
self._setup.gen.h2("Runtimes: conditions")
for runtime_pkg in spack.repo.PATH.packages_with_tags("runtime"):
self._setup.gen.fact(fn.runtime(runtime_pkg))
self._setup.gen.fact(fn.possible_in_link_run(runtime_pkg))
self._setup.gen.newline()
# Inject version rules for runtimes (versions are declared based
# on the available compilers)
self._setup.pkg_version_rules(runtime_pkg)
for imposed_spec, when_spec in self.runtime_conditions:
msg = f"{when_spec} requires {imposed_spec} at runtime"
_ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg)
self._setup.trigger_rules()
self._setup.effect_rules()
class SpecBuilder:
"""Class with actions to rebuild a spec from ASP results."""

42
lib/spack/spack/test/concretize_compiler_runtimes.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import pytest
import spack.paths
import spack.repo
import spack.solver.asp
import spack.spec
from spack.version import Version
pytestmark = [pytest.mark.only_clingo("Original concretizer does not support compiler runtimes")]
@pytest.fixture
def runtime_repo(config):
repo = os.path.join(spack.paths.repos_path, "compiler_runtime.test")
with spack.repo.use_repositories(repo) as mock_repo:
yield mock_repo
@pytest.fixture
def enable_runtimes():
original = spack.solver.asp.WITH_RUNTIME
spack.solver.asp.WITH_RUNTIME = True
yield
spack.solver.asp.WITH_RUNTIME = original
def test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo, enable_runtimes):
s = spack.spec.Spec("a%gcc@10.2.1 ^b%gcc@4.5.0").concretized()
a, b = s["a"], s["b"]
# Both a and b should depend on the same gcc-runtime directly
assert a.dependencies("gcc-runtime") == b.dependencies("gcc-runtime")
# And the gcc-runtime version should be that of the newest gcc used in the dag.
assert a["gcc-runtime"].version == Version("10.2.1")

6
lib/spack/spack/test/conftest.py
View File

@ -44,6 +44,7 @@
import spack.paths
import spack.platforms
import spack.repo
import spack.solver.asp
import spack.stage
import spack.store
import spack.subprocess_context
@ -57,11 +58,6 @@
from spack.util.pattern import Bunch
@pytest.fixture(scope="session", autouse=True)
def drop_gcc_runtime():
spack.package_base.WITH_GCC_RUNTIME = False
def ensure_configuration_fixture_run_before(request):
"""Ensure that fixture mutating the configuration run before the one where
the function is called.

72
var/spack/repos/builtin/packages/gcc-runtime/package.py
View File

@ -20,6 +20,8 @@ class GccRuntime(Package):
homepage = "https://gcc.gnu.org"
has_code = False
tags = ["runtime"]
maintainers("haampie")
license("GPL-3.0-or-later WITH GCC-exception-3.1")
@ -42,76 +44,6 @@ class GccRuntime(Package):
"ubsan",
]
for v in [
"13.2",
"13.1",
"12.3",
"12.2",
"12.1",
"11.4",
"11.3",
"11.2",
"11.1",
"10.5",
"10.4",
"10.3",
"10.2",
"10.1",
"9.5",
"9.4",
"9.3",
"9.2",
"9.1",
"8.5",
"8.4",
"8.3",
"8.2",
"8.1",
"7.5",
"7.4",
"7.3",
"7.2",
"7.1",
"6.5",
"6.4",
"6.3",
"6.2",
"6.1",
"5.5",
"5.4",
"5.3",
"5.2",
"5.1",
"4.9.4",
"4.9.3",
"4.9.2",
"4.9.1",
"4.9.0",
"4.8.5",
"4.8.4",
"4.8.3",
"4.8.2",
"4.8.1",
"4.8.0",
"4.7.4",
"4.7.3",
"4.7.2",
"4.7.1",
"4.7.0",
"4.6.4",
"4.6.3",
"4.6.2",
"4.6.1",
"4.6.0",
"4.5.4",
"4.5.3",
"4.5.2",
"4.5.1",
"4.5.0",
]:
version(v)
requires(f"%gcc@{v}", when=f"@{v}")
def install(self, spec, prefix):
if spec.platform in ["linux", "cray", "freebsd"]:
libraries = self._get_libraries_elf()

29
var/spack/repos/builtin/packages/gcc/package.py
View File

@ -1111,3 +1111,32 @@ def detect_gdc(self):
),
),
)
@classmethod
def runtime_constraints(cls, *, compiler, pkg):
"""Callback function to inject runtime-related rules into the solver.
Rule-injection is obtained through method calls of the ``pkg`` argument.
Documentation for this function is temporary. When the API will be in its final state,
we'll document the behavior at https://spack.readthedocs.io/en/latest/
Args:
compiler: compiler object (node attribute) currently considered
pkg: object used to forward information to the solver
"""
pkg("*").depends_on(
"gcc-runtime",
when="%gcc",
type="link",
description="If any package uses %gcc, it depends on gcc-runtime",
)
pkg("*").depends_on(
f"gcc-runtime@{str(compiler.version)}:",
when=f"%{str(compiler.spec)}",
type="link",
description=f"If any package uses %{str(compiler.spec)}, "
f"it depends on gcc-runtime@{str(compiler.version)}:",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")

13
var/spack/repos/compiler_runtime.test/packages/a/package.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class A(Package):
homepage = "http://www.example.com"
has_code = False
version("1.0")
depends_on("b")

12
var/spack/repos/compiler_runtime.test/packages/b/package.py Normal file
View File

@ -0,0 +1,12 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class B(Package):
homepage = "http://www.example.com"
has_code = False
version("1.0")

13
var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class GccRuntime(Package):
homepage = "https://example.com"
has_code = False
tags = ["runtime"]
requires("%gcc")

32
var/spack/repos/compiler_runtime.test/packages/gcc/package.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class Gcc(Package):
homepage = "http://www.example.com/"
has_code = False
version("13.2.0")
version("12.3.0")
@classmethod
def runtime_constraints(cls, *, compiler, pkg):
pkg("*").depends_on(
"gcc-runtime",
when="%gcc",
type="link",
description="If any package uses %gcc, it depends on gcc-runtime",
)
pkg("*").depends_on(
f"gcc-runtime@{str(compiler.version)}:",
when=f"%{str(compiler.spec)}",
type="link",
description=f"If any package uses %{str(compiler.spec)}, "
f"it depends on gcc-runtime@{str(compiler.version)}:",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")

2
var/spack/repos/compiler_runtime.test/repo.yaml Normal file
View File

@ -0,0 +1,2 @@
repo:
namespace: compiler_runtime.test