Add when context manager to group common constraints in packages (#24650)

This PR adds a context manager that permit to group the common part of a `when=` argument and add that to the context:
```python
class Gcc(AutotoolsPackage):
    with when('+nvptx'):
        depends_on('cuda')
        conflicts('@:6', msg='NVPTX only supported in gcc 7 and above')
        conflicts('languages=ada')
        conflicts('languages=brig')
        conflicts('languages=go')
```
The above snippet is equivalent to:
```python
class Gcc(AutotoolsPackage):
    depends_on('cuda', when='+nvptx')
    conflicts('@:6', when='+nvptx', msg='NVPTX only supported in gcc 7 and above')
    conflicts('languages=ada', when='+nvptx')
    conflicts('languages=brig', when='+nvptx')
    conflicts('languages=go', when='+nvptx')
```
which needs a repetition of the `when='+nvptx'` argument. The context manager might help improving readability and permits to group together directives related to the same semantic aspect (e.g. all the directives needed to model the behavior of `gcc` when `+nvptx` is active). 

Modifications:

- [x] Added a `when` context manager to be used with package directives
- [x] Add unit tests and documentation for the new feature
- [x] Modified `cp2k` and `gcc` to show the use of the context manager
This commit is contained in:
Massimiliano Culpo 2021-07-02 17:43:15 +02:00 committed by GitHub
parent f88d90e432
commit 3d11716e54
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 331 additions and 178 deletions

View File

@ -1257,7 +1257,7 @@ Variants
Many software packages can be configured to enable optional
features, which often come at the expense of additional dependencies or
longer build times. To be flexible enough and support a wide variety of
use cases, Spack permits to expose to the end-user the ability to choose
use cases, Spack allows you to expose to the end-user the ability to choose
which features should be activated in a package at the time it is installed.
The mechanism to be employed is the :py:func:`spack.directives.variant` directive.
@ -2775,6 +2775,57 @@ packages be built with MVAPICH and GCC.
See the :ref:`concretization-preferences` section for more details.
.. _group_when_spec:
----------------------------
Common ``when=`` constraints
----------------------------
In case a package needs many directives to share the whole ``when=``
argument, or just part of it, Spack allows you to group the common part
under a context manager:
.. code-block:: python
class Gcc(AutotoolsPackage):
with when('+nvptx'):
depends_on('cuda')
conflicts('@:6', msg='NVPTX only supported in gcc 7 and above')
conflicts('languages=ada')
conflicts('languages=brig')
conflicts('languages=go')
The snippet above is equivalent to the more verbose:
.. code-block:: python
class Gcc(AutotoolsPackage):
depends_on('cuda', when='+nvptx')
conflicts('@:6', when='+nvptx', msg='NVPTX only supported in gcc 7 and above')
conflicts('languages=ada', when='+nvptx')
conflicts('languages=brig', when='+nvptx')
conflicts('languages=go', when='+nvptx')
Constraints stemming from the context are added to what is explicitly present in the
``when=`` argument of a directive, so:
.. code-block:: python
with when('+elpa'):
depends_on('elpa+openmp', when='+openmp')
is equivalent to:
.. code-block:: python
depends_on('elpa+openmp', when='+openmp+elpa')
Constraints from nested context managers are also added together, but they are rarely
needed or recommended.
.. _install-method:
------------------

View File

@ -27,24 +27,22 @@ class OpenMpi(Package):
* ``version``
"""
import functools
import os.path
import re
import sys
from typing import List, Set # novm
from six import string_types
from typing import Set, List # novm
import six
import llnl.util.lang
import llnl.util.tty.color
import spack.error
import spack.patch
import spack.spec
import spack.url
import spack.variant
from spack.dependency import Dependency, default_deptype, canonical_deptype
from spack.dependency import Dependency, canonical_deptype, default_deptype
from spack.fetch_strategy import from_kwargs
from spack.resource import Resource
from spack.version import Version, VersionChecksumError
@ -114,6 +112,7 @@ class DirectiveMeta(type):
# Set of all known directives
_directive_names = set() # type: Set[str]
_directives_to_be_executed = [] # type: List[str]
_when_constraints_from_context = [] # type: List[str]
def __new__(cls, name, bases, attr_dict):
# Initialize the attribute containing the list of directives
@ -167,6 +166,16 @@ def __init__(cls, name, bases, attr_dict):
super(DirectiveMeta, cls).__init__(name, bases, attr_dict)
@staticmethod
def push_to_context(when_spec):
"""Add a spec to the context constraints."""
DirectiveMeta._when_constraints_from_context.append(when_spec)
@staticmethod
def pop_from_context():
"""Pop the last constraint from the context"""
return DirectiveMeta._when_constraints_from_context.pop()
@staticmethod
def directive(dicts=None):
"""Decorator for Spack directives.
@ -205,15 +214,16 @@ class Foo(Package):
This is just a modular way to add storage attributes to the
Package class, and it's how Spack gets information from the
packages to the core.
"""
global __all__
if isinstance(dicts, string_types):
if isinstance(dicts, six.string_types):
dicts = (dicts, )
if not isinstance(dicts, Sequence):
message = "dicts arg must be list, tuple, or string. Found {0}"
raise TypeError(message.format(type(dicts)))
# Add the dictionary names if not already there
DirectiveMeta._directive_names |= set(dicts)
@ -223,6 +233,23 @@ def _decorator(decorated_function):
@functools.wraps(decorated_function)
def _wrapper(*args, **kwargs):
# Inject when arguments from the context
if DirectiveMeta._when_constraints_from_context:
# Check that directives not yet supporting the when= argument
# are not used inside the context manager
if decorated_function.__name__ in ('version', 'variant'):
msg = ('directive "{0}" cannot be used within a "when"'
' context since it does not support a "when=" '
'argument')
msg = msg.format(decorated_function.__name__)
raise DirectiveError(msg)
when_spec_from_context = ' '.join(
DirectiveMeta._when_constraints_from_context
)
when_spec = kwargs.get('when', '') + ' ' + when_spec_from_context
kwargs['when'] = when_spec
# If any of the arguments are executors returned by a
# directive passed as an argument, don't execute them
# lazily. Instead, let the called directive handle them.
@ -331,7 +358,7 @@ def _depends_on(pkg, spec, when=None, type=default_deptype, patches=None):
patches = [patches]
# auto-call patch() directive on any strings in patch list
patches = [patch(p) if isinstance(p, string_types) else p
patches = [patch(p) if isinstance(p, six.string_types) else p
for p in patches]
assert all(callable(p) for p in patches)

View File

@ -24,14 +24,12 @@
depending on the scenario, regular old conditionals might be clearer,
so package authors should use their judgement.
"""
import functools
import inspect
from llnl.util.lang import caller_locals
import spack.architecture
import spack.directives
import spack.error
from llnl.util.lang import caller_locals
from spack.spec import Spec
@ -156,8 +154,24 @@ def __call__(self, package_self, *args, **kwargs):
class when(object):
def __init__(self, condition):
"""Can be used both as a decorator, for multimethods, or as a context
manager to group ``when=`` arguments together.
Examples are given in the docstrings below.
Args:
condition (str): condition to be met
"""
if isinstance(condition, bool):
self.spec = Spec() if condition else None
else:
self.spec = Spec(condition)
def __call__(self, method):
"""This annotation lets packages declare multiple versions of
methods like install() that depend on the package's spec.
For example:
.. code-block:: python
@ -214,14 +228,6 @@ def install(self, prefix):
platform-specific versions. There's not much we can do to get
around this because of the way decorators work.
"""
def __init__(self, condition):
if isinstance(condition, bool):
self.spec = Spec() if condition else None
else:
self.spec = Spec(condition)
def __call__(self, method):
# In Python 2, Get the first definition of the method in the
# calling scope by looking at the caller's locals. In Python 3,
# we handle this using MultiMethodMeta.__prepare__.
@ -238,6 +244,32 @@ def __call__(self, method):
return original_method
def __enter__(self):
"""Inject the constraint spec into the `when=` argument of directives
in the context.
This context manager allows you to write:
with when('+nvptx'):
conflicts('@:6', msg='NVPTX only supported from gcc 7')
conflicts('languages=ada')
conflicts('languages=brig')
instead of writing:
conflicts('@:6', when='+nvptx', msg='NVPTX only supported from gcc 7')
conflicts('languages=ada', when='+nvptx')
conflicts('languages=brig', when='+nvptx')
Context managers can be nested (but this is not recommended for readability)
and add their constraint to whatever may be already present in the directive
`when=` argument.
"""
spack.directives.DirectiveMeta.push_to_context(str(self.spec))
def __exit__(self, exc_type, exc_val, exc_tb):
spack.directives.DirectiveMeta.pop_from_context()
class MultiMethodError(spack.error.SpackError):
"""Superclass for multimethod dispatch errors"""

View File

@ -32,3 +32,13 @@ def test_true_directives_exist(mock_packages):
assert cls.patches
assert Spec() in cls.patches
def test_constraints_from_context(mock_packages):
pkg_cls = spack.repo.path.get_pkg_class('with-constraint-met')
assert pkg_cls.dependencies
assert Spec('@1.0') in pkg_cls.dependencies['b']
assert pkg_cls.conflicts
assert (Spec('@1.0'), None) in pkg_cls.conflicts['%gcc']

View File

@ -0,0 +1,20 @@
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class WithConstraintMet(Package):
"""Package that tests True when specs on directives."""
homepage = "http://www.example.com"
url = "http://www.example.com/example-1.0.tar.gz"
version('2.0', '0123456789abcdef0123456789abcdef')
version('1.0', '0123456789abcdef0123456789abcdef')
with when('@1.0'):
depends_on('b')
conflicts('%gcc')

View File

@ -2,10 +2,9 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import copy
import os
import os.path
import copy
import spack.util.environment
@ -68,7 +67,7 @@ class Cp2k(MakefilePackage, CudaPackage):
variant('lmax',
description='Maximum supported angular momentum (HFX and others)',
default='5',
values=tuple(map(str, HFX_LMAX_RANGE)),
values=[str(x) for x in HFX_LMAX_RANGE],
multi=False)
depends_on('python', type='build')
@ -78,71 +77,88 @@ class Cp2k(MakefilePackage, CudaPackage):
depends_on('lapack')
depends_on('fftw-api@3')
with when('smm=libxsmm'):
# require libxsmm-1.11+ since 1.10 can leak file descriptors in Fortran
depends_on('libxsmm@1.11:~header-only', when='smm=libxsmm')
depends_on('libxsmm@1.11:~header-only')
# use pkg-config (support added in libxsmm-1.10) to link to libxsmm
depends_on('pkgconfig', type='build', when='smm=libxsmm')
# ... and in CP2K 7.0+ for linking to libint2
depends_on('pkgconfig', type='build', when='+libint@7.0:')
depends_on('pkgconfig', type='build', when='+libxc@7.0:')
depends_on('pkgconfig', type='build')
# please set variants: smm=blas by configuring packages.yaml or install
# cp2k with option smm=blas on aarch64
conflicts('target=aarch64:', msg='libxsmm is not available on arm')
with when('+libint'):
# ... and in CP2K 7.0+ for linking to libint2
depends_on('pkgconfig', type='build', when='@7.0:')
# libint & libxc are always statically linked
depends_on('libint@1.1.4:1.2', when='+libint@3.0:6.9', type='build')
depends_on('libint@1.1.4:1.2', when='@3.0:6.9')
for lmax in HFX_LMAX_RANGE:
# libint2 can be linked dynamically again
depends_on('libint@2.6.0:+fortran tune=cp2k-lmax-{0}'.format(lmax),
when='+libint@7.0: lmax={0}'.format(lmax))
when='@7.0: lmax={0}'.format(lmax))
depends_on('libxc@2.2.2:3.99.0', when='+libxc@:5.5999', type='build')
depends_on('libxc@4.0.3:4.99.0', when='+libxc@6.0:6.9', type='build')
depends_on('libxc@4.0.3:4.99.0', when='+libxc@7.0:8.1')
depends_on('libxc@5.1.3:5.1.99', when='+libxc@8.2:')
with when('+libxc'):
depends_on('pkgconfig', type='build', when='@7.0:')
depends_on('libxc@2.2.2:3.99.0', when='@:5.5999', type='build')
depends_on('libxc@4.0.3:4.99.0', when='@6.0:6.9', type='build')
depends_on('libxc@4.0.3:4.99.0', when='@7.0:8.1')
depends_on('libxc@5.1.3:5.1.99', when='@8.2:')
depends_on('mpi@2:', when='+mpi')
depends_on('scalapack', when='+mpi')
depends_on('cosma+scalapack', when='+cosma')
depends_on('cosma+cuda+scalapack', when='+cosma+cuda')
depends_on('elpa@2011.12:2016.13+openmp', when='+openmp+elpa@:5.999')
depends_on('elpa@2011.12:2017.11+openmp', when='+openmp+elpa@6.0:6.999')
depends_on('elpa@2018.05:+openmp', when='+openmp+elpa@7.0:')
depends_on('elpa@2011.12:2016.13~openmp', when='~openmp+elpa@:5.999')
depends_on('elpa@2011.12:2017.11~openmp', when='~openmp+elpa@6.0:6.999')
depends_on('elpa@2018.05:~openmp', when='~openmp+elpa@7.0:')
depends_on('plumed+shared+mpi', when='+plumed+mpi')
depends_on('plumed+shared~mpi', when='+plumed~mpi')
with when('+mpi'):
depends_on('mpi@2:')
depends_on('scalapack')
with when('+cosma'):
depends_on('cosma+scalapack')
depends_on('cosma+cuda', when='+cuda')
conflicts('~mpi')
# COSMA support was introduced in 8+
conflicts('@:7.999')
with when('+elpa'):
conflicts('~mpi', msg='elpa requires MPI')
depends_on('elpa+openmp', when='+openmp')
depends_on('elpa~openmp', when='~openmp')
depends_on('elpa@2011.12:2016.13', when='@:5.999')
depends_on('elpa@2011.12:2017.11', when='@6.0:6.999')
depends_on('elpa@2018.05:', when='@7.0:')
with when('+plumed'):
depends_on('plumed+shared')
depends_on('plumed+mpi', when='+mpi')
depends_on('plumed~mpi', when='~mpi')
# while we link statically against PEXSI, its own deps may be linked in
# dynamically, therefore can't set this as pure build-type dependency.
depends_on('pexsi+fortran@0.9.0:0.9.999', when='+pexsi@:4.999')
depends_on('pexsi+fortran@0.10.0:', when='+pexsi@5.0:')
with when('+pexsi'):
conflicts('~mpi', msg='pexsi requires MPI')
depends_on('pexsi+fortran@0.9.0:0.9.999', when='@:4.999')
depends_on('pexsi+fortran@0.10.0:', when='@5.0:')
# only OpenMP should be consistenly used, all other common things
# only OpenMP should be consistently used, all other common things
# like ELPA, SCALAPACK are independent and Spack will ensure that
# a consistent/compat. combination is pulled in to the dependency graph.
depends_on('sirius@:6.999+fortran+vdwxc+shared+openmp', when='@:7.999+sirius+openmp')
depends_on('sirius@:6.999+fortran+vdwxc+shared~openmp', when='@:7.999+sirius~openmp')
# a consistent/compatible combination is pulled into the dependency graph.
with when('+sirius'):
depends_on('sirius+fortran+vdwxc+shared')
depends_on('sirius+openmp', when='+openmp')
depends_on('sirius~openmp', when='~openmp')
depends_on('sirius@:6.999', when='@:7.999')
depends_on('sirius@7:', when='@8:')
conflicts('~mpi')
# sirius support was introduced in 7+
conflicts('@:6.999')
depends_on('sirius@7:+fortran+vdwxc+shared+openmp', when='@8:+sirius+openmp')
with when('+libvori'):
depends_on('libvori@201219:', when='@8.1', type='build')
depends_on('libvori@210412:', when='@8.2:', type='build')
# libvori support was introduced in 8+
conflicts('@:7.999')
# the bundled libcusmm uses numpy in the parameter prediction (v7+)
# which is written using Python 3
depends_on('py-numpy', when='@7:+cuda', type='build')
depends_on('python@3.6:', when='@7:+cuda', type='build')
depends_on('libvori@201219:', when='@8.1+libvori', type='build')
depends_on('libvori@210412:', when='@8.2:+libvori', type='build')
depends_on('spglib', when='+spglib')
# PEXSI, ELPA, COSMA and SIRIUS depend on MPI
conflicts('~mpi', '+pexsi')
conflicts('~mpi', '+elpa')
conflicts('~mpi', '+sirius')
conflicts('~mpi', '+cosma')
conflicts('+sirius', '@:6.999') # sirius support was introduced in 7+
conflicts('+cosma', '@:7.999') # COSMA support was introduced in 8+
conflicts('+libvori', '@:7.999') # libvori support was introduced in 8+
conflicts('~cuda', '+cuda_fft')
conflicts('~cuda', '+cuda_blas')
@ -156,10 +172,6 @@ class Cp2k(MakefilePackage, CudaPackage):
conflicts('%clang')
conflicts('%nag')
# please set variants: smm=blas by configuring packages.yaml or install
# cp2k with option smm=blas on aarch64
conflicts('smm=libxsmm', when='target=aarch64:', msg='libxsmm is not available on arm')
conflicts('^fftw~openmp', when='+openmp')
conflicts('^amdfftw~openmp', when='+openmp')
conflicts('^openblas threads=none', when='+openmp')

View File

@ -11,8 +11,7 @@
import llnl.util.tty as tty
import spack.architecture
import spack.util.executable
from spack.operating_systems.mac_os import macos_version, macos_sdk_path
from spack.operating_systems.mac_os import macos_sdk_path, macos_version
class Gcc(AutotoolsPackage, GNUMirrorPackage):
@ -25,7 +24,7 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
list_url = 'http://ftp.gnu.org/gnu/gcc/'
list_depth = 1
maintainers = ['michaelkuhn']
maintainers = ['michaelkuhn', 'alalazo']
version('master', branch='master')
@ -120,11 +119,13 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
# GCC 5.4 https://github.com/spack/spack/issues/6902#issuecomment-433072097
# GCC 7.3 https://github.com/spack/spack/issues/6902#issuecomment-433030376
# GCC 9+ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86724
depends_on('isl@0.14', when='@5.0:5.2 +graphite')
depends_on('isl@0.15', when='@5.3:5.9 +graphite')
depends_on('isl@0.15:0.18', when='@6:8.9 +graphite')
depends_on('isl@0.15:0.20', when='@9:9.9 +graphite')
depends_on('isl@0.15:', when='@10: +graphite')
with when('+graphite'):
depends_on('isl@0.14', when='@5.0:5.2')
depends_on('isl@0.15', when='@5.3:5.9')
depends_on('isl@0.15:0.18', when='@6:8.9')
depends_on('isl@0.15:0.20', when='@9:9.9')
depends_on('isl@0.15:', when='@10:')
depends_on('zlib', when='@6:')
# GCC only tries to link with -lzstd but it requires
# -pthread too when linking against libzstd.a, so
@ -135,26 +136,10 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
depends_on('gnat', when='languages=ada')
depends_on('binutils+gas+ld+plugins~libiberty', when='+binutils', type=('build', 'link', 'run'))
depends_on('zip', type='build', when='languages=java')
depends_on('cuda', when='+nvptx')
# The server is sometimes a bit slow to respond
timeout = {'timeout': 60}
resource(name='newlib',
url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz',
sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b',
destination='newlibsource',
when='+nvptx',
fetch_options=timeout)
# nvptx-tools does not seem to work as a dependency,
# but does fine when the source is inside the gcc build directory
# nvptx-tools doesn't have any releases, so grabbing the last commit
resource(name='nvptx-tools',
git='https://github.com/MentorEmbedded/nvptx-tools',
commit='d0524fbdc86dfca068db5a21cc78ac255b335be5',
when='+nvptx')
# TODO: integrate these libraries.
# depends_on('ppl')
# depends_on('cloog')
@ -167,14 +152,20 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
depends_on('guile@1.4.1:', type='test')
# See https://golang.org/doc/install/gccgo#Releases
provides('golang', when='languages=go @4.6:')
provides('golang@:1', when='languages=go @4.7.1:')
provides('golang@:1.1', when='languages=go @4.8:')
provides('golang@:1.1.2', when='languages=go @4.8.2:')
provides('golang@:1.2', when='languages=go @4.9:')
provides('golang@:1.4', when='languages=go @5:')
provides('golang@:1.6.1', when='languages=go @6:')
provides('golang@:1.8', when='languages=go @7:')
with when('languages=go'):
provides('golang', when='@4.6:')
provides('golang@:1', when='@4.7.1:')
provides('golang@:1.1', when='@4.8:')
provides('golang@:1.1.2', when='@4.8.2:')
provides('golang@:1.2', when='@4.9:')
provides('golang@:1.4', when='@5:')
provides('golang@:1.6.1', when='@6:')
provides('golang@:1.8', when='@7:')
# GCC 4.6 added support for the Go programming language.
# See https://gcc.gnu.org/gcc-4.6/changes.html
conflicts('@:4.5', msg='support for Go has been added in GCC 4.6')
# Go is not supported on macOS
conflicts('platform=darwin', msg='Go not supported on MacOS')
# For a list of valid languages for a specific release,
# run the following command in the GCC source directory:
@ -194,13 +185,6 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
# but this is the first version that accepts 'c' as a valid language.
conflicts('languages=c', when='@:4.7')
# GCC 4.6 added support for the Go programming language.
# See https://gcc.gnu.org/gcc-4.6/changes.html
conflicts('languages=go', when='@:4.5')
# Go is not supported on macOS
conflicts('languages=go', when='platform=darwin')
# The GCC Java frontend and associated libjava runtime library
# have been removed from GCC as of GCC 7.
# See https://gcc.gnu.org/gcc-7/changes.html
@ -210,17 +194,34 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
# See https://gcc.gnu.org/gcc-5/changes.html
conflicts('languages=jit', when='@:4')
with when('+nvptx'):
depends_on('cuda')
resource(
name='newlib',
url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz',
sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b',
destination='newlibsource',
fetch_options=timeout
)
# nvptx-tools does not seem to work as a dependency,
# but does fine when the source is inside the gcc build directory
# nvptx-tools doesn't have any releases, so grabbing the last commit
resource(
name='nvptx-tools',
git='https://github.com/MentorEmbedded/nvptx-tools',
commit='d0524fbdc86dfca068db5a21cc78ac255b335be5',
)
# NVPTX offloading supported in 7 and later by limited languages
conflicts('+nvptx', when='@:6', msg='NVPTX only supported in gcc 7 and above')
conflicts('languages=ada', when='+nvptx')
conflicts('languages=brig', when='+nvptx')
conflicts('languages=go', when='+nvptx')
conflicts('languages=java', when='+nvptx')
conflicts('languages=jit', when='+nvptx')
conflicts('languages=objc', when='+nvptx')
conflicts('languages=obj-c++', when='+nvptx')
conflicts('@:6', msg='NVPTX only supported in gcc 7 and above')
conflicts('languages=ada')
conflicts('languages=brig')
conflicts('languages=go')
conflicts('languages=java')
conflicts('languages=jit')
conflicts('languages=objc')
conflicts('languages=obj-c++')
# NVPTX build disables bootstrap
conflicts('+bootstrap', when='+nvptx')
conflicts('+bootstrap')
# Binutils can't build ld on macOS
conflicts('+binutils', when='platform=darwin')