zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a "sticky" property to variants (#28630)

Browse Source 
* Add sticky variants

* Add unit tests for sticky variants

* Add documentation for sticky variants

* Revert "Revert 19736 because conflicts are avoided by clingo by default (#26721)"

This reverts commit 33ef7d57c1.

* Add stickiness to "allow-unsupported-compiler"
This commit is contained in:
Massimiliano Culpo
2022-02-02 19:05:24 +01:00
committed by GitHub
parent dd7acecf3d
commit cd04109e17
10 changed files with 188 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/spack/docs/packaging_guide.rst
View File

@@ -1441,6 +1441,32 @@ The ``when`` clause follows the same syntax and accepts the same
values as the ``when`` argument of values as the ``when`` argument of
:py:func:`spack.directives.depends_on` :py:func:`spack.directives.depends_on`
^^^^^^^^^^^^^^^
Sticky Variants
^^^^^^^^^^^^^^^
The variant directive can be marked as ``sticky`` by setting to ``True`` the
corresponding argument:
.. code-block:: python
variant('bar', default=False, sticky=True)
A ``sticky`` variant differs from a regular one in that it is always set
to either:
#. An explicit value appearing in a spec literal or
#. Its default value
The concretizer thus is not free to pick an alternate value to work
around conflicts, but will error out instead.
Setting this property on a variant is useful in cases where the
variant allows some dangerous or controversial options (e.g. using unsupported versions
of a compiler for a library) and the packager wants to ensure that
allowing these options is done on purpose by the user, rather than
automatically by the solver.
^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
Overriding Variants Overriding Variants
^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^

179
lib/spack/spack/build_systems/cuda.py
View File

@@ -5,6 +5,7 @@
import spack.variant import spack.variant
from spack.directives import conflicts, depends_on, variant from spack.directives import conflicts, depends_on, variant
from spack.multimethod import when
from spack.package import PackageBase from spack.package import PackageBase
@@ -88,103 +89,103 @@ def cuda_flags(arch_list):
# Linux x86_64 compiler conflicts from here: # Linux x86_64 compiler conflicts from here:
# https://gist.github.com/ax3l/9489132 # https://gist.github.com/ax3l/9489132
with when('^cuda~allow-unsupported-compilers'):
# GCC
# According to
# https://github.com/spack/spack/pull/25054#issuecomment-886531664
# these conflicts are valid independently from the architecture
# GCC # minimum supported versions
# According to conflicts('%gcc@:4', when='+cuda ^cuda@11.0:')
# https://github.com/spack/spack/pull/25054#issuecomment-886531664 conflicts('%gcc@:5', when='+cuda ^cuda@11.4:')
# these conflicts are valid independently from the architecture
# minimum supported versions # maximum supported version
conflicts('%gcc@:4', when='+cuda ^cuda@11.0:') # NOTE:
conflicts('%gcc@:5', when='+cuda ^cuda@11.4:') # in order to not constrain future cuda version to old gcc versions,
# it has been decided to use an upper bound for the latest version.
# This implies that the last one in the list has to be updated at
# each release of a new cuda minor version.
conflicts('%gcc@10:', when='+cuda ^cuda@:11.0')
conflicts('%gcc@12:', when='+cuda ^cuda@:11.6')
conflicts('%clang@13:', when='+cuda ^cuda@:11.5')
conflicts('%clang@14:', when='+cuda ^cuda@:11.6')
# maximum supported version # https://gist.github.com/ax3l/9489132#gistcomment-3860114
# NOTE: conflicts('%gcc@10', when='+cuda ^cuda@:11.4.0')
# in order to not constrain future cuda version to old gcc versions, conflicts('%gcc@5:', when='+cuda ^cuda@:7.5 target=x86_64:')
# it has been decided to use an upper bound for the latest version. conflicts('%gcc@6:', when='+cuda ^cuda@:8 target=x86_64:')
# This implies that the last one in the list has to be updated at conflicts('%gcc@7:', when='+cuda ^cuda@:9.1 target=x86_64:')
# each release of a new cuda minor version. conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130 target=x86_64:')
conflicts('%gcc@10:', when='+cuda ^cuda@:11.0') conflicts('%gcc@9:', when='+cuda ^cuda@:10.2.89 target=x86_64:')
conflicts('%gcc@12:', when='+cuda ^cuda@:11.6') conflicts('%pgi@:14.8', when='+cuda ^cuda@:7.0.27 target=x86_64:')
conflicts('%clang@13:', when='+cuda ^cuda@:11.5') conflicts('%pgi@:15.3,15.5:', when='+cuda ^cuda@7.5 target=x86_64:')
conflicts('%clang@14:', when='+cuda ^cuda@:11.6') conflicts('%pgi@:16.2,16.0:16.3', when='+cuda ^cuda@8 target=x86_64:')
conflicts('%pgi@:15,18:', when='+cuda ^cuda@9.0:9.1 target=x86_64:')
conflicts('%pgi@:16,19:', when='+cuda ^cuda@9.2.88:10 target=x86_64:')
conflicts('%pgi@:17,20:', when='+cuda ^cuda@10.1.105:10.2.89 target=x86_64:')
conflicts('%pgi@:17,21:', when='+cuda ^cuda@11.0.2:11.1.0 target=x86_64:')
conflicts('%clang@:3.4', when='+cuda ^cuda@:7.5 target=x86_64:')
conflicts('%clang@:3.7,4:', when='+cuda ^cuda@8.0:9.0 target=x86_64:')
conflicts('%clang@:3.7,4.1:', when='+cuda ^cuda@9.1 target=x86_64:')
conflicts('%clang@:3.7,5.1:', when='+cuda ^cuda@9.2 target=x86_64:')
conflicts('%clang@:3.7,6.1:', when='+cuda ^cuda@10.0.130 target=x86_64:')
conflicts('%clang@:3.7,7.1:', when='+cuda ^cuda@10.1.105 target=x86_64:')
conflicts('%clang@:3.7,8.1:',
when='+cuda ^cuda@10.1.105:10.1.243 target=x86_64:')
conflicts('%clang@:3.2,9:', when='+cuda ^cuda@10.2.89 target=x86_64:')
conflicts('%clang@:5', when='+cuda ^cuda@11.0.2: target=x86_64:')
conflicts('%clang@10:', when='+cuda ^cuda@:11.0.3 target=x86_64:')
conflicts('%clang@11:', when='+cuda ^cuda@:11.1.0 target=x86_64:')
# https://gist.github.com/ax3l/9489132#gistcomment-3860114 # x86_64 vs. ppc64le differ according to NVidia docs
conflicts('%gcc@10', when='+cuda ^cuda@:11.4.0') # Linux ppc64le compiler conflicts from Table from the docs below:
conflicts('%gcc@5:', when='+cuda ^cuda@:7.5 target=x86_64:') # https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
conflicts('%gcc@6:', when='+cuda ^cuda@:8 target=x86_64:') # https://docs.nvidia.com/cuda/archive/9.2/cuda-installation-guide-linux/index.html
conflicts('%gcc@7:', when='+cuda ^cuda@:9.1 target=x86_64:') # https://docs.nvidia.com/cuda/archive/9.1/cuda-installation-guide-linux/index.html
conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130 target=x86_64:') # https://docs.nvidia.com/cuda/archive/9.0/cuda-installation-guide-linux/index.html
conflicts('%gcc@9:', when='+cuda ^cuda@:10.2.89 target=x86_64:') # https://docs.nvidia.com/cuda/archive/8.0/cuda-installation-guide-linux/index.html
conflicts('%pgi@:14.8', when='+cuda ^cuda@:7.0.27 target=x86_64:')
conflicts('%pgi@:15.3,15.5:', when='+cuda ^cuda@7.5 target=x86_64:')
conflicts('%pgi@:16.2,16.0:16.3', when='+cuda ^cuda@8 target=x86_64:')
conflicts('%pgi@:15,18:', when='+cuda ^cuda@9.0:9.1 target=x86_64:')
conflicts('%pgi@:16,19:', when='+cuda ^cuda@9.2.88:10 target=x86_64:')
conflicts('%pgi@:17,20:', when='+cuda ^cuda@10.1.105:10.2.89 target=x86_64:')
conflicts('%pgi@:17,21:', when='+cuda ^cuda@11.0.2:11.1.0 target=x86_64:')
conflicts('%clang@:3.4', when='+cuda ^cuda@:7.5 target=x86_64:')
conflicts('%clang@:3.7,4:', when='+cuda ^cuda@8.0:9.0 target=x86_64:')
conflicts('%clang@:3.7,4.1:', when='+cuda ^cuda@9.1 target=x86_64:')
conflicts('%clang@:3.7,5.1:', when='+cuda ^cuda@9.2 target=x86_64:')
conflicts('%clang@:3.7,6.1:', when='+cuda ^cuda@10.0.130 target=x86_64:')
conflicts('%clang@:3.7,7.1:', when='+cuda ^cuda@10.1.105 target=x86_64:')
conflicts('%clang@:3.7,8.1:',
when='+cuda ^cuda@10.1.105:10.1.243 target=x86_64:')
conflicts('%clang@:3.2,9:', when='+cuda ^cuda@10.2.89 target=x86_64:')
conflicts('%clang@:5', when='+cuda ^cuda@11.0.2: target=x86_64:')
conflicts('%clang@10:', when='+cuda ^cuda@:11.0.3 target=x86_64:')
conflicts('%clang@11:', when='+cuda ^cuda@:11.1.0 target=x86_64:')
# x86_64 vs. ppc64le differ according to NVidia docs # information prior to CUDA 9 difficult to find
# Linux ppc64le compiler conflicts from Table from the docs below: conflicts('%gcc@6:', when='+cuda ^cuda@:9 target=ppc64le:')
# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130 target=ppc64le:')
# https://docs.nvidia.com/cuda/archive/9.2/cuda-installation-guide-linux/index.html conflicts('%gcc@9:', when='+cuda ^cuda@:10.1.243 target=ppc64le:')
# https://docs.nvidia.com/cuda/archive/9.1/cuda-installation-guide-linux/index.html # officially, CUDA 11.0.2 only supports the system GCC 8.3 on ppc64le
# https://docs.nvidia.com/cuda/archive/9.0/cuda-installation-guide-linux/index.html conflicts('%pgi', when='+cuda ^cuda@:8 target=ppc64le:')
# https://docs.nvidia.com/cuda/archive/8.0/cuda-installation-guide-linux/index.html conflicts('%pgi@:16', when='+cuda ^cuda@:9.1.185 target=ppc64le:')
conflicts('%pgi@:17', when='+cuda ^cuda@:10 target=ppc64le:')
conflicts('%clang@4:', when='+cuda ^cuda@:9.0.176 target=ppc64le:')
conflicts('%clang@5:', when='+cuda ^cuda@:9.1 target=ppc64le:')
conflicts('%clang@6:', when='+cuda ^cuda@:9.2 target=ppc64le:')
conflicts('%clang@7:', when='+cuda ^cuda@10.0.130 target=ppc64le:')
conflicts('%clang@7.1:', when='+cuda ^cuda@:10.1.105 target=ppc64le:')
conflicts('%clang@8.1:', when='+cuda ^cuda@:10.2.89 target=ppc64le:')
conflicts('%clang@:5', when='+cuda ^cuda@11.0.2: target=ppc64le:')
conflicts('%clang@10:', when='+cuda ^cuda@:11.0.2 target=ppc64le:')
conflicts('%clang@11:', when='+cuda ^cuda@:11.1.0 target=ppc64le:')
# information prior to CUDA 9 difficult to find # Intel is mostly relevant for x86_64 Linux, even though it also
conflicts('%gcc@6:', when='+cuda ^cuda@:9 target=ppc64le:') # exists for Mac OS X. No information prior to CUDA 3.2 or Intel 11.1
conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130 target=ppc64le:') conflicts('%intel@:11.0', when='+cuda ^cuda@:3.1')
conflicts('%gcc@9:', when='+cuda ^cuda@:10.1.243 target=ppc64le:') conflicts('%intel@:12.0', when='+cuda ^cuda@5.5:')
# officially, CUDA 11.0.2 only supports the system GCC 8.3 on ppc64le conflicts('%intel@:13.0', when='+cuda ^cuda@6.0:')
conflicts('%pgi', when='+cuda ^cuda@:8 target=ppc64le:') conflicts('%intel@:13.2', when='+cuda ^cuda@6.5:')
conflicts('%pgi@:16', when='+cuda ^cuda@:9.1.185 target=ppc64le:') conflicts('%intel@:14.9', when='+cuda ^cuda@7:')
conflicts('%pgi@:17', when='+cuda ^cuda@:10 target=ppc64le:') # Intel 15.x is compatible with CUDA 7 thru current CUDA
conflicts('%clang@4:', when='+cuda ^cuda@:9.0.176 target=ppc64le:') conflicts('%intel@16.0:', when='+cuda ^cuda@:8.0.43')
conflicts('%clang@5:', when='+cuda ^cuda@:9.1 target=ppc64le:') conflicts('%intel@17.0:', when='+cuda ^cuda@:8.0.60')
conflicts('%clang@6:', when='+cuda ^cuda@:9.2 target=ppc64le:') conflicts('%intel@18.0:', when='+cuda ^cuda@:9.9')
conflicts('%clang@7:', when='+cuda ^cuda@10.0.130 target=ppc64le:') conflicts('%intel@19.0:', when='+cuda ^cuda@:10.0')
conflicts('%clang@7.1:', when='+cuda ^cuda@:10.1.105 target=ppc64le:') conflicts('%intel@19.1:', when='+cuda ^cuda@:10.1')
conflicts('%clang@8.1:', when='+cuda ^cuda@:10.2.89 target=ppc64le:') conflicts('%intel@19.2:', when='+cuda ^cuda@:11.1.0')
conflicts('%clang@:5', when='+cuda ^cuda@11.0.2: target=ppc64le:')
conflicts('%clang@10:', when='+cuda ^cuda@:11.0.2 target=ppc64le:')
conflicts('%clang@11:', when='+cuda ^cuda@:11.1.0 target=ppc64le:')
# Intel is mostly relevant for x86_64 Linux, even though it also # XL is mostly relevant for ppc64le Linux
# exists for Mac OS X. No information prior to CUDA 3.2 or Intel 11.1 conflicts('%xl@:12,14:', when='+cuda ^cuda@:9.1')
conflicts('%intel@:11.0', when='+cuda ^cuda@:3.1') conflicts('%xl@:12,14:15,17:', when='+cuda ^cuda@9.2')
conflicts('%intel@:12.0', when='+cuda ^cuda@5.5:') conflicts('%xl@:12,17:', when='+cuda ^cuda@:11.1.0')
conflicts('%intel@:13.0', when='+cuda ^cuda@6.0:')
conflicts('%intel@:13.2', when='+cuda ^cuda@6.5:')
conflicts('%intel@:14.9', when='+cuda ^cuda@7:')
# Intel 15.x is compatible with CUDA 7 thru current CUDA
conflicts('%intel@16.0:', when='+cuda ^cuda@:8.0.43')
conflicts('%intel@17.0:', when='+cuda ^cuda@:8.0.60')
conflicts('%intel@18.0:', when='+cuda ^cuda@:9.9')
conflicts('%intel@19.0:', when='+cuda ^cuda@:10.0')
conflicts('%intel@19.1:', when='+cuda ^cuda@:10.1')
conflicts('%intel@19.2:', when='+cuda ^cuda@:11.1.0')
# XL is mostly relevant for ppc64le Linux # Darwin.
conflicts('%xl@:12,14:', when='+cuda ^cuda@:9.1') # TODO: add missing conflicts for %apple-clang cuda@:10
conflicts('%xl@:12,14:15,17:', when='+cuda ^cuda@9.2') conflicts('platform=darwin', when='+cuda ^cuda@11.0.2: ')
conflicts('%xl@:12,17:', when='+cuda ^cuda@:11.1.0')
# Darwin.
# TODO: add missing conflicts for %apple-clang cuda@:10
conflicts('platform=darwin', when='+cuda ^cuda@11.0.2: ')
# Make sure cuda_arch can not be used without +cuda # Make sure cuda_arch can not be used without +cuda
for value in cuda_arch_values: for value in cuda_arch_values:

9
lib/spack/spack/directives.py
View File

@@ -562,7 +562,9 @@ def variant(
values=None, values=None,
multi=None, multi=None,
validator=None, validator=None,
when=None): when=None,
sticky=False
):
"""Define a variant for the package. Packager can specify a default """Define a variant for the package. Packager can specify a default
value as well as a text description. value as well as a text description.
@@ -583,7 +585,8 @@ def variant(
doesn't meet the additional constraints doesn't meet the additional constraints
when (spack.spec.Spec, bool): optional condition on which the when (spack.spec.Spec, bool): optional condition on which the
variant applies variant applies
sticky (bool): the variant should not be changed by the concretizer to
find a valid concrete spec.
Raises: Raises:
DirectiveError: if arguments passed to the directive are invalid DirectiveError: if arguments passed to the directive are invalid
""" """
@@ -657,7 +660,7 @@ def _execute_variant(pkg):
when_specs += orig_when when_specs += orig_when
pkg.variants[name] = (spack.variant.Variant( pkg.variants[name] = (spack.variant.Variant(
name, default, description, values, multi, validator name, default, description, values, multi, validator, sticky
), when_specs) ), when_specs)
return _execute_variant return _execute_variant

3
lib/spack/spack/solver/asp.py
View File

@@ -852,6 +852,9 @@ def pkg_rules(self, pkg, tests):
for value in sorted(values): for value in sorted(values):
self.gen.fact(fn.variant_possible_value(pkg.name, name, value)) self.gen.fact(fn.variant_possible_value(pkg.name, name, value))
if variant.sticky:
self.gen.fact(fn.variant_sticky(pkg.name, name))
self.gen.newline() self.gen.newline()
# conflicts # conflicts

9
lib/spack/spack/solver/concretize.lp
View File

@@ -402,6 +402,14 @@ variant(Package, Variant) :- variant_condition(ID, Package, Variant),
build(Package), build(Package),
error("Unsatisfied conditional variants cannot take on a variant value"). error("Unsatisfied conditional variants cannot take on a variant value").
% if a variant is sticky and not set its value is the default value
variant_value(Package, Variant, Value) :-
variant(Package, Variant),
not variant_set(Package, Variant),
variant_sticky(Package, Variant),
variant_default_value(Package, Variant, Value),
build(Package).
% one variant value for single-valued variants. % one variant value for single-valued variants.
1 { 1 {
variant_value(Package, Variant, Value) variant_value(Package, Variant, Value)
@@ -523,6 +531,7 @@ variant_single_value(Package, "dev_path")
% spec or some package sets it, and without this, clingo will give % spec or some package sets it, and without this, clingo will give
% warnings like 'info: atom does not occur in any rule head'. % warnings like 'info: atom does not occur in any rule head'.
#defined variant/2. #defined variant/2.
#defined variant_sticky/2.
#defined variant_set/3. #defined variant_set/3.
#defined variant_condition/3. #defined variant_condition/3.
#defined variant_single_value/2. #defined variant_single_value/2.

18
lib/spack/spack/test/concretize.py
View File

@@ -1357,3 +1357,21 @@ def test_concrete_specs_are_not_modified_on_reuse(
s = spack.spec.Spec(spec_str).concretized(reuse=True) s = spack.spec.Spec(spec_str).concretized(reuse=True)
assert s.package.installed is expect_installed assert s.package.installed is expect_installed
assert s.satisfies(spec_str, strict=True) assert s.satisfies(spec_str, strict=True)
@pytest.mark.regression('26721,19736')
def test_sticky_variant_in_package(self):
if spack.config.get('config:concretizer') == 'original':
pytest.skip('Original concretizer cannot use sticky variants')
# Here we test that a sticky variant cannot be changed from its default value
# by the ASP solver if not set explicitly. The package used in the test needs
# to have +allow-gcc set to be concretized with %gcc and clingo is not allowed
# to change the default ~allow-gcc
with pytest.raises(spack.error.SpackError):
spack.spec.Spec('sticky-variant %gcc').concretized()
s = spack.spec.Spec('sticky-variant+allow-gcc %gcc').concretized()
assert s.satisfies('%gcc') and s.satisfies('+allow-gcc')
s = spack.spec.Spec('sticky-variant %clang').concretized()
assert s.satisfies('%clang') and s.satisfies('~allow-gcc')

10
lib/spack/spack/test/concretize_preferences.py
View File

@@ -416,3 +416,13 @@ def test_multivalued_variants_are_lower_priority_than_providers(self):
): ):
s = Spec('somevirtual').concretized() s = Spec('somevirtual').concretized()
assert s.name == 'some-virtual-preferred' assert s.name == 'some-virtual-preferred'
@pytest.mark.regression('26721,19736')
def test_sticky_variant_accounts_for_packages_yaml(self):
with spack.config.override(
'packages:sticky-variant', {
'variants': '+allow-gcc'
}
):
s = Spec('sticky-variant %gcc').concretized()
assert s.satisfies('%gcc') and s.satisfies('+allow-gcc')

7
lib/spack/spack/variant.py
View File

@@ -37,7 +37,9 @@ def __init__(
description, description,
values=(True, False), values=(True, False),
multi=False, multi=False,
validator=None): validator=None,
sticky=False
):
"""Initialize a package variant. """Initialize a package variant.
Args: Args:
@@ -51,6 +53,8 @@ def __init__(
multi (bool): whether multiple CSV are allowed multi (bool): whether multiple CSV are allowed
validator (callable): optional callable used to enforce validator (callable): optional callable used to enforce
additional logic on the set of values being validated additional logic on the set of values being validated
sticky (bool): if true the variant is set to the default value at
concretization time
""" """
self.name = name self.name = name
self.default = default self.default = default
@@ -83,6 +87,7 @@ def isa_type(v):
self.multi = multi self.multi = multi
self.group_validator = validator self.group_validator = validator
self.sticky = sticky
def validate_or_raise(self, vspec, pkg=None): def validate_or_raise(self, vspec, pkg=None):
"""Validate a variant spec against this package variant. Raises an """Validate a variant spec against this package variant. Raises an

18
var/spack/repos/builtin.mock/packages/sticky-variant/package.py Normal file
View File

@@ -0,0 +1,18 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class StickyVariant(AutotoolsPackage):
"""Package with a sticky variant and a conflict"""
homepage = "http://www.example.com"
url = "http://www.example.com/a-1.0.tar.gz"
version('1.0', '0123456789abcdef0123456789abcdef')
variant('allow-gcc', description='', default=False, sticky=True)
conflicts('%gcc', when='~allow-gcc')

2
var/spack/repos/builtin/packages/cuda/package.py
View File

@@ -144,6 +144,8 @@ class Cuda(Package):
conflicts('arch=darwin-mojave-x86_64') conflicts('arch=darwin-mojave-x86_64')
variant('dev', default=False, description='Enable development dependencies, i.e to use cuda-gdb') variant('dev', default=False, description='Enable development dependencies, i.e to use cuda-gdb')
variant('allow-unsupported-compilers', default=False, sticky=True,
description='Allow unsupported host compiler and CUDA version combinations')
depends_on('libxml2', when='@10.1.243:') depends_on('libxml2', when='@10.1.243:')
# cuda-gdb needed libncurses.so.5 before 11.4.0 # cuda-gdb needed libncurses.so.5 before 11.4.0