zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configuration: allow users to enforce hard spec constraints (#27987)

Browse Source 
Spack doesn't have an easy way to say something like "If I build
package X, then I *need* version Y":

* If you specify something on the command line, then you ensure
  that the constraints are applied, but the package is always built
* Likewise if you `spack add X...`` to your environment, the
  constraints are guaranteed to hold, but the environment always
  builds the package
* You can add preferences to packages.yaml, but these are not
  guaranteed to hold (Spack can choose other settings)

This commit adds a 'require' subsection to packages.yaml: the
specs added there are guaranteed to hold. The commit includes
documentation for the feature.

Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>
This commit is contained in:
Peter Scheibel
2022-08-16 11:44:30 -07:00
committed by GitHub
parent 0d981a012d
commit 8281a0c5fe
7 changed files with 545 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
lib/spack/spack/schema/packages.py
View File

@@ -21,6 +21,29 @@
"default": {},
"additionalProperties": False,
"properties": {
"require": {
"oneOf": [
# 'require' can be a list of requirement_groups.
# each requirement group is a list of one or more
# specs. Either at least one or exactly one spec
# in the group must be satisfied (depending on
# whether you use "any_of" or "one_of",
# repectively)
{
"type": "array",
"items": {
"type": "object",
"properties": {
"one_of": {"type": "array"},
"any_of": {"type": "array"},
},
},
},
# Shorthand for a single requirement group with
# one member
{"type": "string"},
]
},
"version": {
"type": "array",
"default": [],

26
lib/spack/spack/solver/asp.py
View File

@@ -927,6 +927,30 @@ def package_compiler_defaults(self, pkg):
fn.node_compiler_preference(pkg.name, cspec.name, cspec.version, -i * 100)
)
def package_requirement_rules(self, pkg):
pkg_name = pkg.name
config = spack.config.get("packages")
requirements = config.get(pkg_name, {}).get("require", [])
if isinstance(requirements, string_types):
rules = [(pkg_name, "one_of", [requirements])]
else:
rules = []
for requirement in requirements:
for policy in ("one_of", "any_of"):
if policy in requirement:
rules.append((pkg_name, policy, requirement[policy]))
for requirement_grp_id, (pkg_name, policy, requirement_grp) in enumerate(rules):
self.gen.fact(fn.requirement_group(pkg_name, requirement_grp_id))
self.gen.fact(fn.requirement_policy(pkg_name, requirement_grp_id, policy))
for requirement_weight, spec_str in enumerate(requirement_grp):
spec = spack.spec.Spec(spec_str)
if not spec.name:
spec.name = pkg_name
member_id = self.condition(spec, imposed_spec=spec, name=pkg_name)
self.gen.fact(fn.requirement_group_member(member_id, pkg_name, requirement_grp_id))
self.gen.fact(fn.requirement_has_weight(member_id, requirement_weight))
def pkg_rules(self, pkg, tests):
pkg = packagize(pkg)
@@ -1017,6 +1041,8 @@ def pkg_rules(self, pkg, tests):
lambda v, p, i: self.gen.fact(fn.pkg_provider_preference(pkg.name, v, p, i)),
)
self.package_requirement_rules(pkg)
def condition(self, required_spec, imposed_spec=None, name=None, msg=None):
"""Generate facts for a dependency or virtual provider condition.

178
lib/spack/spack/solver/concretize.lp
View File

@@ -500,6 +500,49 @@ error(2, "Attempted to use external for '{0}' which does not satisfy any configu
#defined external_spec_condition/4.
#defined external_spec_condition/5.
%-----------------------------------------------------------------------------
% Config required semantics
%-----------------------------------------------------------------------------
requirement_group_satisfied(Package, X) :-
1 { condition_holds(Y) : requirement_group_member(Y, Package, X) } 1,
node(Package),
requirement_policy(Package, X, "one_of"),
requirement_group(Package, X).
requirement_weight(Package, W) :-
condition_holds(Y),
requirement_has_weight(Y, W),
requirement_group_member(Y, Package, X),
requirement_policy(Package, X, "one_of"),
requirement_group_satisfied(Package, X).
requirement_group_satisfied(Package, X) :-
1 { condition_holds(Y) : requirement_group_member(Y, Package, X) } ,
node(Package),
requirement_policy(Package, X, "any_of"),
requirement_group(Package, X).
requirement_weight(Package, W) :-
W = #min {
Z : requirement_has_weight(Y, Z), condition_holds(Y), requirement_group_member(Y, Package, X);
% We need this to avoid an annoying warning during the solve
% concretize.lp:1151:5-11: info: tuple ignored:
% #sup@73
10000
},
requirement_policy(Package, X, "any_of"),
requirement_group_satisfied(Package, X).
error(2, "Cannot satisfy requirement group for package '{0}'", Package) :-
node(Package),
requirement_group(Package, X),
not requirement_group_satisfied(Package, X).
#defined requirement_group/2.
#defined requirement_group_member/3.
#defined requirement_has_weight/2.
%-----------------------------------------------------------------------------
% Variant semantics
%-----------------------------------------------------------------------------
@@ -898,8 +941,7 @@ error(2, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package,
% the compiler associated with the node satisfy the same constraint
node_compiler_version_satisfies(Package, Compiler, Constraint)
:- node_compiler_version(Package, Compiler, Version),
compiler_version_satisfies(Compiler, Constraint, Version),
build(Package).
compiler_version_satisfies(Compiler, Constraint, Version).
#defined compiler_version_satisfies/3.
@@ -1092,12 +1134,24 @@ opt_criterion(100, "number of packages to build (vs. reuse)").
#minimize { 1@100,Package : build(Package), optimize_for_reuse() }.
#defined optimize_for_reuse/0.
% A condition group specifies one or more specs that must be satisfied.
% Specs declared first are preferred, so we assign increasing weights and
% minimize the weights.
opt_criterion(75, "requirement weight").
#minimize{ 0@275: #true }.
#minimize{ 0@75: #true }.
#minimize {
Weight@75+Priority
: requirement_weight(Package, Weight),
build_priority(Package, Priority)
}.
% Minimize the number of deprecated versions being used
opt_criterion(15, "deprecated versions used").
#minimize{ 0@215: #true }.
#minimize{ 0@15: #true }.
opt_criterion(73, "deprecated versions used").
#minimize{ 0@273: #true }.
#minimize{ 0@73: #true }.
#minimize{
1@15+Priority,Package
1@73+Priority,Package
: deprecated(Package, _),
build_priority(Package, Priority)
}.
@@ -1106,51 +1160,51 @@ opt_criterion(15, "deprecated versions used").
% 1. Version weight
% 2. Number of variants with a non default value, if not set
% for the root(Package)
opt_criterion(14, "version weight").
#minimize{ 0@214: #true }.
#minimize{ 0@14: #true }.
opt_criterion(70, "version weight").
#minimize{ 0@270: #true }.
#minimize{ 0@70: #true }.
#minimize {
Weight@14+Priority
Weight@70+Priority
: root(Package),version_weight(Package, Weight),
build_priority(Package, Priority)
}.
opt_criterion(13, "number of non-default variants (roots)").
#minimize{ 0@213: #true }.
#minimize{ 0@13: #true }.
opt_criterion(65, "number of non-default variants (roots)").
#minimize{ 0@265: #true }.
#minimize{ 0@65: #true }.
#minimize {
1@13+Priority,Package,Variant,Value
1@65+Priority,Package,Variant,Value
: variant_not_default(Package, Variant, Value),
root(Package),
build_priority(Package, Priority)
}.
opt_criterion(12, "preferred providers for roots").
#minimize{ 0@212 : #true }.
#minimize{ 0@12: #true }.
opt_criterion(60, "preferred providers for roots").
#minimize{ 0@260: #true }.
#minimize{ 0@60: #true }.
#minimize{
Weight@12+Priority,Provider,Virtual
Weight@60+Priority,Provider,Virtual
: provider_weight(Provider, Virtual, Weight),
root(Provider),
build_priority(Provider, Priority)
}.
opt_criterion(11, "default values of variants not being used (roots)").
#minimize{ 0@211: #true }.
#minimize{ 0@11: #true }.
opt_criterion(55, "default values of variants not being used (roots)").
#minimize{ 0@255: #true }.
#minimize{ 0@55: #true }.
#minimize{
1@11+Priority,Package,Variant,Value
1@55+Priority,Package,Variant,Value
: variant_default_not_used(Package, Variant, Value),
root(Package),
build_priority(Package, Priority)
}.
% Try to use default variants or variants that have been set
opt_criterion(10, "number of non-default variants (non-roots)").
#minimize{ 0@210: #true }.
#minimize{ 0@10: #true }.
opt_criterion(50, "number of non-default variants (non-roots)").
#minimize{ 0@250: #true }.
#minimize{ 0@50: #true }.
#minimize {
1@10+Priority,Package,Variant,Value
1@50+Priority,Package,Variant,Value
: variant_not_default(Package, Variant, Value),
not root(Package),
build_priority(Package, Priority)
@@ -1158,91 +1212,91 @@ opt_criterion(10, "number of non-default variants (non-roots)").
% Minimize the weights of the providers, i.e. use as much as
% possible the most preferred providers
opt_criterion(9, "preferred providers (non-roots)").
#minimize{ 0@209: #true }.
#minimize{ 0@9: #true }.
opt_criterion(45, "preferred providers (non-roots)").
#minimize{ 0@245: #true }.
#minimize{ 0@45: #true }.
#minimize{
Weight@9+Priority,Provider,Virtual
Weight@45+Priority,Provider,Virtual
: provider_weight(Provider, Virtual, Weight), not root(Provider),
build_priority(Provider, Priority)
}.
% Try to minimize the number of compiler mismatches in the DAG.
opt_criterion(8, "compiler mismatches").
#minimize{ 0@208: #true }.
#minimize{ 0@8: #true }.
opt_criterion(40, "compiler mismatches").
#minimize{ 0@240: #true }.
#minimize{ 0@40: #true }.
#minimize{
1@8+Priority,Package,Dependency
1@40+Priority,Package,Dependency
: compiler_mismatch(Package, Dependency),
build_priority(Package, Priority)
}.
% Try to minimize the number of compiler mismatches in the DAG.
opt_criterion(7, "OS mismatches").
#minimize{ 0@207: #true }.
#minimize{ 0@7: #true }.
opt_criterion(35, "OS mismatches").
#minimize{ 0@235: #true }.
#minimize{ 0@35: #true }.
#minimize{
1@7+Priority,Package,Dependency
1@35+Priority,Package,Dependency
: node_os_mismatch(Package, Dependency),
build_priority(Package, Priority)
}.
opt_criterion(6, "non-preferred OS's").
#minimize{ 0@206: #true }.
#minimize{ 0@6: #true }.
opt_criterion(30, "non-preferred OS's").
#minimize{ 0@230: #true }.
#minimize{ 0@30: #true }.
#minimize{
Weight@6+Priority,Package
Weight@30+Priority,Package
: node_os_weight(Package, Weight),
build_priority(Package, Priority)
}.
% Choose more recent versions for nodes
opt_criterion(5, "version badness").
#minimize{ 0@205: #true }.
#minimize{ 0@5: #true }.
opt_criterion(25, "version badness").
#minimize{ 0@225: #true }.
#minimize{ 0@25: #true }.
#minimize{
Weight@5+Priority,Package
Weight@25+Priority,Package
: version_weight(Package, Weight),
build_priority(Package, Priority)
}.
% Try to use all the default values of variants
opt_criterion(4, "default values of variants not being used (non-roots)").
#minimize{ 0@204: #true }.
#minimize{ 0@4: #true }.
opt_criterion(20, "default values of variants not being used (non-roots)").
#minimize{ 0@220: #true }.
#minimize{ 0@20: #true }.
#minimize{
1@4+Priority,Package,Variant,Value
1@20+Priority,Package,Variant,Value
: variant_default_not_used(Package, Variant, Value),
not root(Package),
build_priority(Package, Priority)
}.
% Try to use preferred compilers
opt_criterion(3, "non-preferred compilers").
#minimize{ 0@203: #true }.
#minimize{ 0@3: #true }.
opt_criterion(15, "non-preferred compilers").
#minimize{ 0@215: #true }.
#minimize{ 0@15: #true }.
#minimize{
Weight@3+Priority,Package
Weight@15+Priority,Package
: compiler_weight(Package, Weight),
build_priority(Package, Priority)
}.
% Minimize the number of mismatches for targets in the DAG, try
% to select the preferred target.
opt_criterion(2, "target mismatches").
#minimize{ 0@202: #true }.
#minimize{ 0@2: #true }.
opt_criterion(10, "target mismatches").
#minimize{ 0@210: #true }.
#minimize{ 0@10: #true }.
#minimize{
1@2+Priority,Package,Dependency
1@10+Priority,Package,Dependency
: node_target_mismatch(Package, Dependency),
build_priority(Package, Priority)
}.
opt_criterion(1, "non-preferred targets").
#minimize{ 0@201: #true }.
#minimize{ 0@1: #true }.
opt_criterion(5, "non-preferred targets").
#minimize{ 0@205: #true }.
#minimize{ 0@5: #true }.
#minimize{
Weight@1+Priority,Package
Weight@5+Priority,Package
: node_target_weight(Package, Weight),
build_priority(Package, Priority)
}.

14
lib/spack/spack/test/concretize_preferences.py
View File

@@ -12,23 +12,11 @@
import spack.package_prefs
import spack.repo
import spack.util.spack_yaml as syaml
from spack.config import ConfigError, ConfigScope
from spack.config import ConfigError
from spack.spec import Spec
from spack.version import Version
@pytest.fixture()
def concretize_scope(mutable_config, tmpdir):
"""Adds a scope for concretization preferences"""
tmpdir.ensure_dir("concretize")
mutable_config.push_scope(ConfigScope("concretize", str(tmpdir.join("concretize"))))
yield
mutable_config.pop_scope()
spack.repo.path._provider_index = None
@pytest.fixture()
def configure_permissions():
conf = syaml.load_config(

299
lib/spack/spack/test/concretize_requirements.py Normal file
View File

@@ -0,0 +1,299 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import sys
import pytest
import spack.config
import spack.repo
import spack.util.spack_yaml as syaml
from spack.solver.asp import UnsatisfiableSpecError
from spack.spec import Spec
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Windows uses old concretizer")
def update_packages_config(conf_str):
conf = syaml.load_config(conf_str)
spack.config.set("packages", conf["packages"], scope="concretize")
_pkgx = (
"x",
"""\
class X(Package):
version('1.1')
version('1.0')
version('0.9')
variant('shared', default=True,
description='Build shared libraries')
depends_on('y')
""",
)
_pkgy = (
"y",
"""\
class Y(Package):
version('2.5')
version('2.4')
version('2.3', deprecated=True)
variant('shared', default=True,
description='Build shared libraries')
""",
)
_pkgv = (
"v",
"""\
class V(Package):
version('2.1')
version('2.0')
""",
)
@pytest.fixture
def create_test_repo(tmpdir, mutable_config):
repo_path = str(tmpdir)
repo_yaml = tmpdir.join("repo.yaml")
with open(str(repo_yaml), "w") as f:
f.write(
"""\
repo:
namespace: testcfgrequirements
"""
)
packages_dir = tmpdir.join("packages")
for (pkg_name, pkg_str) in [_pkgx, _pkgy, _pkgv]:
pkg_dir = packages_dir.ensure(pkg_name, dir=True)
pkg_file = pkg_dir.join("package.py")
with open(str(pkg_file), "w") as f:
f.write(pkg_str)
yield spack.repo.Repo(repo_path)
@pytest.fixture
def test_repo(create_test_repo, monkeypatch, mock_stage):
with spack.repo.use_repositories(create_test_repo) as mock_repo_path:
yield mock_repo_path
class MakeStage(object):
def __init__(self, stage):
self.stage = stage
def __call__(self, *args, **kwargs):
return self.stage
@pytest.fixture
def fake_installs(monkeypatch, tmpdir):
stage_path = str(tmpdir.ensure("fake-stage", dir=True))
universal_unused_stage = spack.stage.DIYStage(stage_path)
monkeypatch.setattr(
spack.package_base.Package, "_make_stage", MakeStage(universal_unused_stage)
)
def test_requirement_isnt_optional(concretize_scope, test_repo):
"""If a user spec requests something that directly conflicts
with a requirement, make sure we get an error.
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
x:
require: "@1.0"
"""
update_packages_config(conf_str)
with pytest.raises(UnsatisfiableSpecError):
Spec("x@1.1").concretize()
def test_requirement_is_successfully_applied(concretize_scope, test_repo):
"""If a simple requirement can be satisfied, make sure the
concretization succeeds and the requirement spec is applied.
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
s1 = Spec("x").concretized()
# Without any requirements/preferences, the later version is preferred
assert s1.satisfies("@1.1")
conf_str = """\
packages:
x:
require: "@1.0"
"""
update_packages_config(conf_str)
s2 = Spec("x").concretized()
# The requirement forces choosing the eariler version
assert s2.satisfies("@1.0")
def test_multiple_packages_requirements_are_respected(concretize_scope, test_repo):
"""Apply requirements to two packages; make sure the concretization
succeeds and both requirements are respected.
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
x:
require: "@1.0"
y:
require: "@2.4"
"""
update_packages_config(conf_str)
spec = Spec("x").concretized()
assert spec["x"].satisfies("@1.0")
assert spec["y"].satisfies("@2.4")
def test_oneof(concretize_scope, test_repo):
"""'one_of' allows forcing the concretizer to satisfy one of
the specs in the group (but not all have to be satisfied).
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
y:
require:
- one_of: ["@2.4", "~shared"]
"""
update_packages_config(conf_str)
spec = Spec("x").concretized()
# The concretizer only has to satisfy one of @2.4/~shared, and @2.4
# comes first so it is prioritized
assert spec["y"].satisfies("@2.4+shared")
def test_one_package_multiple_oneof_groups(concretize_scope, test_repo):
"""One package has two 'one_of' groups; check that both are
applied.
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
y:
require:
- one_of: ["@2.4%gcc", "@2.5%clang"]
- one_of: ["@2.5~shared", "@2.4+shared"]
"""
update_packages_config(conf_str)
s1 = Spec("y@2.5").concretized()
assert s1.satisfies("%clang~shared")
s2 = Spec("y@2.4").concretized()
assert s2.satisfies("%gcc+shared")
def test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo):
"""Specify requirements for specs that are not concretized or
a dependency of a concretized spec (in other words, none of
the requirements are used for the requested spec).
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
# Note that the exact contents aren't important since this isn't
# intended to be used, but the important thing is that a number of
# packages have requirements applied
conf_str = """\
packages:
x:
require: "@1.0"
y:
require:
- one_of: ["@2.4%gcc", "@2.5%clang"]
- one_of: ["@2.5~shared", "@2.4+shared"]
"""
update_packages_config(conf_str)
s1 = Spec("v").concretized()
assert s1.satisfies("@2.1")
def test_oneof_ordering(concretize_scope, test_repo):
"""Ensure that earlier elements of 'one_of' have higher priority.
This priority should override default priority (e.g. choosing
later versions).
"""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
y:
require:
- one_of: ["@2.4", "@2.5"]
"""
update_packages_config(conf_str)
s1 = Spec("y").concretized()
assert s1.satisfies("@2.4")
s2 = Spec("y@2.5").concretized()
assert s2.satisfies("@2.5")
def test_reuse_oneof(concretize_scope, create_test_repo, mutable_database, fake_installs):
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
conf_str = """\
packages:
y:
require:
- one_of: ["@2.5", "%gcc"]
"""
with spack.repo.use_repositories(create_test_repo):
s1 = Spec("y@2.5%gcc").concretized()
s1.package.do_install(fake=True, explicit=True)
update_packages_config(conf_str)
with spack.config.override("concretizer:reuse", True):
s2 = Spec("y").concretized()
assert not s2.satisfies("@2.5 %gcc")
def test_requirements_are_higher_priority_than_deprecation(concretize_scope, test_repo):
"""Test that users can override a deprecated version with a requirement."""
if spack.config.get("config:concretizer") == "original":
pytest.skip("Original concretizer does not support configuration" " requirements")
# @2.3 is a deprecated versions. Ensure that any_of picks both constraints,
# since they are possible
conf_str = """\
packages:
y:
require:
- any_of: ["@2.3", "%gcc"]
"""
update_packages_config(conf_str)
s1 = Spec("y").concretized()
assert s1.satisfies("@2.3")
assert s1.satisfies("%gcc")

14
lib/spack/spack/test/conftest.py
View File

@@ -706,6 +706,20 @@ def mutable_empty_config(tmpdir_factory, configuration_dir):
yield cfg
@pytest.fixture(scope="function")
def concretize_scope(mutable_config, tmpdir):
"""Adds a scope for concretization preferences"""
tmpdir.ensure_dir("concretize")
mutable_config.push_scope(
spack.config.ConfigScope("concretize", str(tmpdir.join("concretize")))
)
yield
mutable_config.pop_scope()
spack.repo.path._provider_index = None
@pytest.fixture
def no_compilers_yaml(mutable_config):
"""Creates a temporary configuration without compilers.yaml"""